This is page 29 of 47. Use http://codebase.md/doobidoo/mcp-memory-service?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .claude
│ ├── agents
│ │ ├── amp-bridge.md
│ │ ├── amp-pr-automator.md
│ │ ├── code-quality-guard.md
│ │ ├── gemini-pr-automator.md
│ │ └── github-release-manager.md
│ ├── settings.local.json.backup
│ └── settings.local.json.local
├── .commit-message
├── .dockerignore
├── .env.example
├── .env.sqlite.backup
├── .envnn#
├── .gitattributes
├── .github
│ ├── FUNDING.yml
│ ├── ISSUE_TEMPLATE
│ │ ├── bug_report.yml
│ │ ├── config.yml
│ │ ├── feature_request.yml
│ │ └── performance_issue.yml
│ ├── pull_request_template.md
│ └── workflows
│ ├── bridge-tests.yml
│ ├── CACHE_FIX.md
│ ├── claude-code-review.yml
│ ├── claude.yml
│ ├── cleanup-images.yml.disabled
│ ├── dev-setup-validation.yml
│ ├── docker-publish.yml
│ ├── LATEST_FIXES.md
│ ├── main-optimized.yml.disabled
│ ├── main.yml
│ ├── publish-and-test.yml
│ ├── README_OPTIMIZATION.md
│ ├── release-tag.yml.disabled
│ ├── release.yml
│ ├── roadmap-review-reminder.yml
│ ├── SECRET_CONDITIONAL_FIX.md
│ └── WORKFLOW_FIXES.md
├── .gitignore
├── .mcp.json.backup
├── .mcp.json.template
├── .pyscn
│ ├── .gitignore
│ └── reports
│ └── analyze_20251123_214224.html
├── AGENTS.md
├── archive
│ ├── deployment
│ │ ├── deploy_fastmcp_fixed.sh
│ │ ├── deploy_http_with_mcp.sh
│ │ └── deploy_mcp_v4.sh
│ ├── deployment-configs
│ │ ├── empty_config.yml
│ │ └── smithery.yaml
│ ├── development
│ │ └── test_fastmcp.py
│ ├── docs-removed-2025-08-23
│ │ ├── authentication.md
│ │ ├── claude_integration.md
│ │ ├── claude-code-compatibility.md
│ │ ├── claude-code-integration.md
│ │ ├── claude-code-quickstart.md
│ │ ├── claude-desktop-setup.md
│ │ ├── complete-setup-guide.md
│ │ ├── database-synchronization.md
│ │ ├── development
│ │ │ ├── autonomous-memory-consolidation.md
│ │ │ ├── CLEANUP_PLAN.md
│ │ │ ├── CLEANUP_README.md
│ │ │ ├── CLEANUP_SUMMARY.md
│ │ │ ├── dream-inspired-memory-consolidation.md
│ │ │ ├── hybrid-slm-memory-consolidation.md
│ │ │ ├── mcp-milestone.md
│ │ │ ├── multi-client-architecture.md
│ │ │ ├── test-results.md
│ │ │ └── TIMESTAMP_FIX_SUMMARY.md
│ │ ├── distributed-sync.md
│ │ ├── invocation_guide.md
│ │ ├── macos-intel.md
│ │ ├── master-guide.md
│ │ ├── mcp-client-configuration.md
│ │ ├── multi-client-server.md
│ │ ├── service-installation.md
│ │ ├── sessions
│ │ │ └── MCP_ENHANCEMENT_SESSION_MEMORY_v4.1.0.md
│ │ ├── UBUNTU_SETUP.md
│ │ ├── ubuntu.md
│ │ ├── windows-setup.md
│ │ └── windows.md
│ ├── docs-root-cleanup-2025-08-23
│ │ ├── AWESOME_LIST_SUBMISSION.md
│ │ ├── CLOUDFLARE_IMPLEMENTATION.md
│ │ ├── DOCUMENTATION_ANALYSIS.md
│ │ ├── DOCUMENTATION_CLEANUP_PLAN.md
│ │ ├── DOCUMENTATION_CONSOLIDATION_COMPLETE.md
│ │ ├── LITESTREAM_SETUP_GUIDE.md
│ │ ├── lm_studio_system_prompt.md
│ │ ├── PYTORCH_DOWNLOAD_FIX.md
│ │ └── README-ORIGINAL-BACKUP.md
│ ├── investigations
│ │ └── MACOS_HOOKS_INVESTIGATION.md
│ ├── litestream-configs-v6.3.0
│ │ ├── install_service.sh
│ │ ├── litestream_master_config_fixed.yml
│ │ ├── litestream_master_config.yml
│ │ ├── litestream_replica_config_fixed.yml
│ │ ├── litestream_replica_config.yml
│ │ ├── litestream_replica_simple.yml
│ │ ├── litestream-http.service
│ │ ├── litestream.service
│ │ └── requirements-cloudflare.txt
│ ├── release-notes
│ │ └── release-notes-v7.1.4.md
│ └── setup-development
│ ├── README.md
│ ├── setup_consolidation_mdns.sh
│ ├── STARTUP_SETUP_GUIDE.md
│ └── test_service.sh
├── CHANGELOG-HISTORIC.md
├── CHANGELOG.md
├── claude_commands
│ ├── memory-context.md
│ ├── memory-health.md
│ ├── memory-ingest-dir.md
│ ├── memory-ingest.md
│ ├── memory-recall.md
│ ├── memory-search.md
│ ├── memory-store.md
│ ├── README.md
│ └── session-start.md
├── claude-hooks
│ ├── config.json
│ ├── config.template.json
│ ├── CONFIGURATION.md
│ ├── core
│ │ ├── memory-retrieval.js
│ │ ├── mid-conversation.js
│ │ ├── session-end.js
│ │ ├── session-start.js
│ │ └── topic-change.js
│ ├── debug-pattern-test.js
│ ├── install_claude_hooks_windows.ps1
│ ├── install_hooks.py
│ ├── memory-mode-controller.js
│ ├── MIGRATION.md
│ ├── README-NATURAL-TRIGGERS.md
│ ├── README-phase2.md
│ ├── README.md
│ ├── simple-test.js
│ ├── statusline.sh
│ ├── test-adaptive-weights.js
│ ├── test-dual-protocol-hook.js
│ ├── test-mcp-hook.js
│ ├── test-natural-triggers.js
│ ├── test-recency-scoring.js
│ ├── tests
│ │ ├── integration-test.js
│ │ ├── phase2-integration-test.js
│ │ ├── test-code-execution.js
│ │ ├── test-cross-session.json
│ │ ├── test-session-tracking.json
│ │ └── test-threading.json
│ ├── utilities
│ │ ├── adaptive-pattern-detector.js
│ │ ├── context-formatter.js
│ │ ├── context-shift-detector.js
│ │ ├── conversation-analyzer.js
│ │ ├── dynamic-context-updater.js
│ │ ├── git-analyzer.js
│ │ ├── mcp-client.js
│ │ ├── memory-client.js
│ │ ├── memory-scorer.js
│ │ ├── performance-manager.js
│ │ ├── project-detector.js
│ │ ├── session-tracker.js
│ │ ├── tiered-conversation-monitor.js
│ │ └── version-checker.js
│ └── WINDOWS-SESSIONSTART-BUG.md
├── CLAUDE.md
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── Development-Sprint-November-2025.md
├── docs
│ ├── amp-cli-bridge.md
│ ├── api
│ │ ├── code-execution-interface.md
│ │ ├── memory-metadata-api.md
│ │ ├── PHASE1_IMPLEMENTATION_SUMMARY.md
│ │ ├── PHASE2_IMPLEMENTATION_SUMMARY.md
│ │ ├── PHASE2_REPORT.md
│ │ └── tag-standardization.md
│ ├── architecture
│ │ ├── search-enhancement-spec.md
│ │ └── search-examples.md
│ ├── architecture.md
│ ├── archive
│ │ └── obsolete-workflows
│ │ ├── load_memory_context.md
│ │ └── README.md
│ ├── assets
│ │ └── images
│ │ ├── dashboard-v3.3.0-preview.png
│ │ ├── memory-awareness-hooks-example.png
│ │ ├── project-infographic.svg
│ │ └── README.md
│ ├── CLAUDE_CODE_QUICK_REFERENCE.md
│ ├── cloudflare-setup.md
│ ├── deployment
│ │ ├── docker.md
│ │ ├── dual-service.md
│ │ ├── production-guide.md
│ │ └── systemd-service.md
│ ├── development
│ │ ├── ai-agent-instructions.md
│ │ ├── code-quality
│ │ │ ├── phase-2a-completion.md
│ │ │ ├── phase-2a-handle-get-prompt.md
│ │ │ ├── phase-2a-index.md
│ │ │ ├── phase-2a-install-package.md
│ │ │ └── phase-2b-session-summary.md
│ │ ├── code-quality-workflow.md
│ │ ├── dashboard-workflow.md
│ │ ├── issue-management.md
│ │ ├── pr-review-guide.md
│ │ ├── refactoring-notes.md
│ │ ├── release-checklist.md
│ │ └── todo-tracker.md
│ ├── docker-optimized-build.md
│ ├── document-ingestion.md
│ ├── DOCUMENTATION_AUDIT.md
│ ├── enhancement-roadmap-issue-14.md
│ ├── examples
│ │ ├── analysis-scripts.js
│ │ ├── maintenance-session-example.md
│ │ ├── memory-distribution-chart.jsx
│ │ └── tag-schema.json
│ ├── first-time-setup.md
│ ├── glama-deployment.md
│ ├── guides
│ │ ├── advanced-command-examples.md
│ │ ├── chromadb-migration.md
│ │ ├── commands-vs-mcp-server.md
│ │ ├── mcp-enhancements.md
│ │ ├── mdns-service-discovery.md
│ │ ├── memory-consolidation-guide.md
│ │ ├── migration.md
│ │ ├── scripts.md
│ │ └── STORAGE_BACKENDS.md
│ ├── HOOK_IMPROVEMENTS.md
│ ├── hooks
│ │ └── phase2-code-execution-migration.md
│ ├── http-server-management.md
│ ├── ide-compatability.md
│ ├── IMAGE_RETENTION_POLICY.md
│ ├── images
│ │ └── dashboard-placeholder.md
│ ├── implementation
│ │ ├── health_checks.md
│ │ └── performance.md
│ ├── IMPLEMENTATION_PLAN_HTTP_SSE.md
│ ├── integration
│ │ ├── homebrew.md
│ │ └── multi-client.md
│ ├── integrations
│ │ ├── gemini.md
│ │ ├── groq-bridge.md
│ │ ├── groq-integration-summary.md
│ │ └── groq-model-comparison.md
│ ├── integrations.md
│ ├── legacy
│ │ └── dual-protocol-hooks.md
│ ├── LM_STUDIO_COMPATIBILITY.md
│ ├── maintenance
│ │ └── memory-maintenance.md
│ ├── mastery
│ │ ├── api-reference.md
│ │ ├── architecture-overview.md
│ │ ├── configuration-guide.md
│ │ ├── local-setup-and-run.md
│ │ ├── testing-guide.md
│ │ └── troubleshooting.md
│ ├── migration
│ │ └── code-execution-api-quick-start.md
│ ├── natural-memory-triggers
│ │ ├── cli-reference.md
│ │ ├── installation-guide.md
│ │ └── performance-optimization.md
│ ├── oauth-setup.md
│ ├── pr-graphql-integration.md
│ ├── quick-setup-cloudflare-dual-environment.md
│ ├── README.md
│ ├── remote-configuration-wiki-section.md
│ ├── research
│ │ ├── code-execution-interface-implementation.md
│ │ └── code-execution-interface-summary.md
│ ├── ROADMAP.md
│ ├── sqlite-vec-backend.md
│ ├── statistics
│ │ ├── charts
│ │ │ ├── activity_patterns.png
│ │ │ ├── contributors.png
│ │ │ ├── growth_trajectory.png
│ │ │ ├── monthly_activity.png
│ │ │ └── october_sprint.png
│ │ ├── data
│ │ │ ├── activity_by_day.csv
│ │ │ ├── activity_by_hour.csv
│ │ │ ├── contributors.csv
│ │ │ └── monthly_activity.csv
│ │ ├── generate_charts.py
│ │ └── REPOSITORY_STATISTICS.md
│ ├── technical
│ │ ├── development.md
│ │ ├── memory-migration.md
│ │ ├── migration-log.md
│ │ ├── sqlite-vec-embedding-fixes.md
│ │ └── tag-storage.md
│ ├── testing
│ │ └── regression-tests.md
│ ├── testing-cloudflare-backend.md
│ ├── troubleshooting
│ │ ├── cloudflare-api-token-setup.md
│ │ ├── cloudflare-authentication.md
│ │ ├── general.md
│ │ ├── hooks-quick-reference.md
│ │ ├── pr162-schema-caching-issue.md
│ │ ├── session-end-hooks.md
│ │ └── sync-issues.md
│ └── tutorials
│ ├── advanced-techniques.md
│ ├── data-analysis.md
│ └── demo-session-walkthrough.md
├── examples
│ ├── claude_desktop_config_template.json
│ ├── claude_desktop_config_windows.json
│ ├── claude-desktop-http-config.json
│ ├── config
│ │ └── claude_desktop_config.json
│ ├── http-mcp-bridge.js
│ ├── memory_export_template.json
│ ├── README.md
│ ├── setup
│ │ └── setup_multi_client_complete.py
│ └── start_https_example.sh
├── install_service.py
├── install.py
├── LICENSE
├── NOTICE
├── pyproject.toml
├── pytest.ini
├── README.md
├── run_server.py
├── scripts
│ ├── .claude
│ │ └── settings.local.json
│ ├── archive
│ │ └── check_missing_timestamps.py
│ ├── backup
│ │ ├── backup_memories.py
│ │ ├── backup_sqlite_vec.sh
│ │ ├── export_distributable_memories.sh
│ │ └── restore_memories.py
│ ├── benchmarks
│ │ ├── benchmark_code_execution_api.py
│ │ ├── benchmark_hybrid_sync.py
│ │ └── benchmark_server_caching.py
│ ├── database
│ │ ├── analyze_sqlite_vec_db.py
│ │ ├── check_sqlite_vec_status.py
│ │ ├── db_health_check.py
│ │ └── simple_timestamp_check.py
│ ├── development
│ │ ├── debug_server_initialization.py
│ │ ├── find_orphaned_files.py
│ │ ├── fix_mdns.sh
│ │ ├── fix_sitecustomize.py
│ │ ├── remote_ingest.sh
│ │ ├── setup-git-merge-drivers.sh
│ │ ├── uv-lock-merge.sh
│ │ └── verify_hybrid_sync.py
│ ├── hooks
│ │ └── pre-commit
│ ├── installation
│ │ ├── install_linux_service.py
│ │ ├── install_macos_service.py
│ │ ├── install_uv.py
│ │ ├── install_windows_service.py
│ │ ├── install.py
│ │ ├── setup_backup_cron.sh
│ │ ├── setup_claude_mcp.sh
│ │ └── setup_cloudflare_resources.py
│ ├── linux
│ │ ├── service_status.sh
│ │ ├── start_service.sh
│ │ ├── stop_service.sh
│ │ ├── uninstall_service.sh
│ │ └── view_logs.sh
│ ├── maintenance
│ │ ├── assign_memory_types.py
│ │ ├── check_memory_types.py
│ │ ├── cleanup_corrupted_encoding.py
│ │ ├── cleanup_memories.py
│ │ ├── cleanup_organize.py
│ │ ├── consolidate_memory_types.py
│ │ ├── consolidation_mappings.json
│ │ ├── delete_orphaned_vectors_fixed.py
│ │ ├── fast_cleanup_duplicates_with_tracking.sh
│ │ ├── find_all_duplicates.py
│ │ ├── find_cloudflare_duplicates.py
│ │ ├── find_duplicates.py
│ │ ├── memory-types.md
│ │ ├── README.md
│ │ ├── recover_timestamps_from_cloudflare.py
│ │ ├── regenerate_embeddings.py
│ │ ├── repair_malformed_tags.py
│ │ ├── repair_memories.py
│ │ ├── repair_sqlite_vec_embeddings.py
│ │ ├── repair_zero_embeddings.py
│ │ ├── restore_from_json_export.py
│ │ └── scan_todos.sh
│ ├── migration
│ │ ├── cleanup_mcp_timestamps.py
│ │ ├── legacy
│ │ │ └── migrate_chroma_to_sqlite.py
│ │ ├── mcp-migration.py
│ │ ├── migrate_sqlite_vec_embeddings.py
│ │ ├── migrate_storage.py
│ │ ├── migrate_tags.py
│ │ ├── migrate_timestamps.py
│ │ ├── migrate_to_cloudflare.py
│ │ ├── migrate_to_sqlite_vec.py
│ │ ├── migrate_v5_enhanced.py
│ │ ├── TIMESTAMP_CLEANUP_README.md
│ │ └── verify_mcp_timestamps.py
│ ├── pr
│ │ ├── amp_collect_results.sh
│ │ ├── amp_detect_breaking_changes.sh
│ │ ├── amp_generate_tests.sh
│ │ ├── amp_pr_review.sh
│ │ ├── amp_quality_gate.sh
│ │ ├── amp_suggest_fixes.sh
│ │ ├── auto_review.sh
│ │ ├── detect_breaking_changes.sh
│ │ ├── generate_tests.sh
│ │ ├── lib
│ │ │ └── graphql_helpers.sh
│ │ ├── quality_gate.sh
│ │ ├── resolve_threads.sh
│ │ ├── run_pyscn_analysis.sh
│ │ ├── run_quality_checks.sh
│ │ ├── thread_status.sh
│ │ └── watch_reviews.sh
│ ├── quality
│ │ ├── fix_dead_code_install.sh
│ │ ├── phase1_dead_code_analysis.md
│ │ ├── phase2_complexity_analysis.md
│ │ ├── README_PHASE1.md
│ │ ├── README_PHASE2.md
│ │ ├── track_pyscn_metrics.sh
│ │ └── weekly_quality_review.sh
│ ├── README.md
│ ├── run
│ │ ├── run_mcp_memory.sh
│ │ ├── run-with-uv.sh
│ │ └── start_sqlite_vec.sh
│ ├── run_memory_server.py
│ ├── server
│ │ ├── check_http_server.py
│ │ ├── check_server_health.py
│ │ ├── memory_offline.py
│ │ ├── preload_models.py
│ │ ├── run_http_server.py
│ │ ├── run_memory_server.py
│ │ ├── start_http_server.bat
│ │ └── start_http_server.sh
│ ├── service
│ │ ├── deploy_dual_services.sh
│ │ ├── install_http_service.sh
│ │ ├── mcp-memory-http.service
│ │ ├── mcp-memory.service
│ │ ├── memory_service_manager.sh
│ │ ├── service_control.sh
│ │ ├── service_utils.py
│ │ └── update_service.sh
│ ├── sync
│ │ ├── check_drift.py
│ │ ├── claude_sync_commands.py
│ │ ├── export_memories.py
│ │ ├── import_memories.py
│ │ ├── litestream
│ │ │ ├── apply_local_changes.sh
│ │ │ ├── enhanced_memory_store.sh
│ │ │ ├── init_staging_db.sh
│ │ │ ├── io.litestream.replication.plist
│ │ │ ├── manual_sync.sh
│ │ │ ├── memory_sync.sh
│ │ │ ├── pull_remote_changes.sh
│ │ │ ├── push_to_remote.sh
│ │ │ ├── README.md
│ │ │ ├── resolve_conflicts.sh
│ │ │ ├── setup_local_litestream.sh
│ │ │ ├── setup_remote_litestream.sh
│ │ │ ├── staging_db_init.sql
│ │ │ ├── stash_local_changes.sh
│ │ │ ├── sync_from_remote_noconfig.sh
│ │ │ └── sync_from_remote.sh
│ │ ├── README.md
│ │ ├── safe_cloudflare_update.sh
│ │ ├── sync_memory_backends.py
│ │ └── sync_now.py
│ ├── testing
│ │ ├── run_complete_test.py
│ │ ├── run_memory_test.sh
│ │ ├── simple_test.py
│ │ ├── test_cleanup_logic.py
│ │ ├── test_cloudflare_backend.py
│ │ ├── test_docker_functionality.py
│ │ ├── test_installation.py
│ │ ├── test_mdns.py
│ │ ├── test_memory_api.py
│ │ ├── test_memory_simple.py
│ │ ├── test_migration.py
│ │ ├── test_search_api.py
│ │ ├── test_sqlite_vec_embeddings.py
│ │ ├── test_sse_events.py
│ │ ├── test-connection.py
│ │ └── test-hook.js
│ ├── utils
│ │ ├── claude_commands_utils.py
│ │ ├── generate_personalized_claude_md.sh
│ │ ├── groq
│ │ ├── groq_agent_bridge.py
│ │ ├── list-collections.py
│ │ ├── memory_wrapper_uv.py
│ │ ├── query_memories.py
│ │ ├── smithery_wrapper.py
│ │ ├── test_groq_bridge.sh
│ │ └── uv_wrapper.py
│ └── validation
│ ├── check_dev_setup.py
│ ├── check_documentation_links.py
│ ├── diagnose_backend_config.py
│ ├── validate_configuration_complete.py
│ ├── validate_memories.py
│ ├── validate_migration.py
│ ├── validate_timestamp_integrity.py
│ ├── verify_environment.py
│ ├── verify_pytorch_windows.py
│ └── verify_torch.py
├── SECURITY.md
├── selective_timestamp_recovery.py
├── SPONSORS.md
├── src
│ └── mcp_memory_service
│ ├── __init__.py
│ ├── api
│ │ ├── __init__.py
│ │ ├── client.py
│ │ ├── operations.py
│ │ ├── sync_wrapper.py
│ │ └── types.py
│ ├── backup
│ │ ├── __init__.py
│ │ └── scheduler.py
│ ├── cli
│ │ ├── __init__.py
│ │ ├── ingestion.py
│ │ ├── main.py
│ │ └── utils.py
│ ├── config.py
│ ├── consolidation
│ │ ├── __init__.py
│ │ ├── associations.py
│ │ ├── base.py
│ │ ├── clustering.py
│ │ ├── compression.py
│ │ ├── consolidator.py
│ │ ├── decay.py
│ │ ├── forgetting.py
│ │ ├── health.py
│ │ └── scheduler.py
│ ├── dependency_check.py
│ ├── discovery
│ │ ├── __init__.py
│ │ ├── client.py
│ │ └── mdns_service.py
│ ├── embeddings
│ │ ├── __init__.py
│ │ └── onnx_embeddings.py
│ ├── ingestion
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── chunker.py
│ │ ├── csv_loader.py
│ │ ├── json_loader.py
│ │ ├── pdf_loader.py
│ │ ├── registry.py
│ │ ├── semtools_loader.py
│ │ └── text_loader.py
│ ├── lm_studio_compat.py
│ ├── mcp_server.py
│ ├── models
│ │ ├── __init__.py
│ │ └── memory.py
│ ├── server.py
│ ├── services
│ │ ├── __init__.py
│ │ └── memory_service.py
│ ├── storage
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── cloudflare.py
│ │ ├── factory.py
│ │ ├── http_client.py
│ │ ├── hybrid.py
│ │ └── sqlite_vec.py
│ ├── sync
│ │ ├── __init__.py
│ │ ├── exporter.py
│ │ ├── importer.py
│ │ └── litestream_config.py
│ ├── utils
│ │ ├── __init__.py
│ │ ├── cache_manager.py
│ │ ├── content_splitter.py
│ │ ├── db_utils.py
│ │ ├── debug.py
│ │ ├── document_processing.py
│ │ ├── gpu_detection.py
│ │ ├── hashing.py
│ │ ├── http_server_manager.py
│ │ ├── port_detection.py
│ │ ├── system_detection.py
│ │ └── time_parser.py
│ └── web
│ ├── __init__.py
│ ├── api
│ │ ├── __init__.py
│ │ ├── analytics.py
│ │ ├── backup.py
│ │ ├── consolidation.py
│ │ ├── documents.py
│ │ ├── events.py
│ │ ├── health.py
│ │ ├── manage.py
│ │ ├── mcp.py
│ │ ├── memories.py
│ │ ├── search.py
│ │ └── sync.py
│ ├── app.py
│ ├── dependencies.py
│ ├── oauth
│ │ ├── __init__.py
│ │ ├── authorization.py
│ │ ├── discovery.py
│ │ ├── middleware.py
│ │ ├── models.py
│ │ ├── registration.py
│ │ └── storage.py
│ ├── sse.py
│ └── static
│ ├── app.js
│ ├── index.html
│ ├── README.md
│ ├── sse_test.html
│ └── style.css
├── start_http_debug.bat
├── start_http_server.sh
├── test_document.txt
├── test_version_checker.js
├── tests
│ ├── __init__.py
│ ├── api
│ │ ├── __init__.py
│ │ ├── test_compact_types.py
│ │ └── test_operations.py
│ ├── bridge
│ │ ├── mock_responses.js
│ │ ├── package-lock.json
│ │ ├── package.json
│ │ └── test_http_mcp_bridge.js
│ ├── conftest.py
│ ├── consolidation
│ │ ├── __init__.py
│ │ ├── conftest.py
│ │ ├── test_associations.py
│ │ ├── test_clustering.py
│ │ ├── test_compression.py
│ │ ├── test_consolidator.py
│ │ ├── test_decay.py
│ │ └── test_forgetting.py
│ ├── contracts
│ │ └── api-specification.yml
│ ├── integration
│ │ ├── package-lock.json
│ │ ├── package.json
│ │ ├── test_api_key_fallback.py
│ │ ├── test_api_memories_chronological.py
│ │ ├── test_api_tag_time_search.py
│ │ ├── test_api_with_memory_service.py
│ │ ├── test_bridge_integration.js
│ │ ├── test_cli_interfaces.py
│ │ ├── test_cloudflare_connection.py
│ │ ├── test_concurrent_clients.py
│ │ ├── test_data_serialization_consistency.py
│ │ ├── test_http_server_startup.py
│ │ ├── test_mcp_memory.py
│ │ ├── test_mdns_integration.py
│ │ ├── test_oauth_basic_auth.py
│ │ ├── test_oauth_flow.py
│ │ ├── test_server_handlers.py
│ │ └── test_store_memory.py
│ ├── performance
│ │ ├── test_background_sync.py
│ │ └── test_hybrid_live.py
│ ├── README.md
│ ├── smithery
│ │ └── test_smithery.py
│ ├── sqlite
│ │ └── simple_sqlite_vec_test.py
│ ├── test_client.py
│ ├── test_content_splitting.py
│ ├── test_database.py
│ ├── test_hybrid_cloudflare_limits.py
│ ├── test_hybrid_storage.py
│ ├── test_memory_ops.py
│ ├── test_semantic_search.py
│ ├── test_sqlite_vec_storage.py
│ ├── test_time_parser.py
│ ├── test_timestamp_preservation.py
│ ├── timestamp
│ │ ├── test_hook_vs_manual_storage.py
│ │ ├── test_issue99_final_validation.py
│ │ ├── test_search_retrieval_inconsistency.py
│ │ ├── test_timestamp_issue.py
│ │ └── test_timestamp_simple.py
│ └── unit
│ ├── conftest.py
│ ├── test_cloudflare_storage.py
│ ├── test_csv_loader.py
│ ├── test_fastapi_dependencies.py
│ ├── test_import.py
│ ├── test_json_loader.py
│ ├── test_mdns_simple.py
│ ├── test_mdns.py
│ ├── test_memory_service.py
│ ├── test_memory.py
│ ├── test_semtools_loader.py
│ ├── test_storage_interface_compatibility.py
│ └── test_tag_time_filtering.py
├── tools
│ ├── docker
│ │ ├── DEPRECATED.md
│ │ ├── docker-compose.http.yml
│ │ ├── docker-compose.pythonpath.yml
│ │ ├── docker-compose.standalone.yml
│ │ ├── docker-compose.uv.yml
│ │ ├── docker-compose.yml
│ │ ├── docker-entrypoint-persistent.sh
│ │ ├── docker-entrypoint-unified.sh
│ │ ├── docker-entrypoint.sh
│ │ ├── Dockerfile
│ │ ├── Dockerfile.glama
│ │ ├── Dockerfile.slim
│ │ ├── README.md
│ │ └── test-docker-modes.sh
│ └── README.md
└── uv.lock
```
# Files
--------------------------------------------------------------------------------
/docs/development/refactoring-notes.md:
--------------------------------------------------------------------------------
```markdown
1 | # Memory Service Refactoring Summary
2 |
3 | ## 2025-02-XX Duplication Review
4 |
5 | - **Memory response serialization** – `src/mcp_memory_service/web/api/memories.py:86` re-implements the same field mapping already provided by `src/mcp_memory_service/services/memory_service.py:83`. We can convert HTTP responses by delegating to `MemoryService.format_memory_response` and avoid keeping two copies of the field list in sync.
6 | - **Search helpers drift** – `src/mcp_memory_service/web/api/search.py:75` and `src/mcp_memory_service/web/api/search.py:84` duplicate logic that now lives inside `MemoryService.retrieve_memory` and `MemoryService.search_by_tag`. The module still defines legacy helpers (`parse_time_query`, `is_within_time_range`) at `src/mcp_memory_service/web/api/search.py:365` that mirror `src/mcp_memory_service/services/memory_service.py:502` and `src/mcp_memory_service/services/memory_service.py:535`; they appear unused and should either call through to the service or be removed.
7 | - **MCP tool vs HTTP MCP API** – Each tool is implemented twice (FastMCP server in `src/mcp_memory_service/mcp_server.py` and HTTP bridge in `src/mcp_memory_service/web/api/mcp.py`), with near-identical request handling and result shaping. Examples: `store_memory` (`mcp_server.py:154` vs `web/api/mcp.py:247`), `retrieve_memory` (`mcp_server.py:204` vs `web/api/mcp.py:282`), `search_by_tag` (`mcp_server.py:262` vs `web/api/mcp.py:313`), `delete_memory` (`mcp_server.py:330` vs `web/api/mcp.py:384`), `check_database_health` (`mcp_server.py:367` vs `web/api/mcp.py:398`), `list_memories` (`mcp_server.py:394` vs `web/api/mcp.py:407`), `search_by_time` (`mcp_server.py:440` vs `web/api/mcp.py:427`), and `search_similar` (`mcp_server.py:502` vs `web/api/mcp.py:463`). Consolidating these into shared helpers would keep the tool surface synchronized and reduce error-prone duplication.
8 |
9 | ## Problem Identified
10 |
11 | The original implementation had **duplicated and inconsistent logic** between the API and MCP tool implementations for `list_memories`:
12 |
13 | ### Critical Issues Found:
14 |
15 | 1. **Different Pagination Logic:**
16 | - **API**: Correctly filters first, then paginates
17 | - **MCP Tool**: Paginates first, then filters (loses data!)
18 |
19 | 2. **Inconsistent Tag Filtering:**
20 | - **API**: Uses `storage.search_by_tag([tag])` for proper tag-based queries
21 | - **MCP Tool**: Uses in-memory filtering after pagination
22 |
23 | 3. **Wrong Total Counts:**
24 | - **API**: Provides accurate `total` and `has_more` for pagination
25 | - **MCP Tool**: Returns incorrect `total_found` count
26 |
27 | 4. **Code Duplication:**
28 | - Same business logic implemented in 3 different places
29 | - Maintenance nightmare and inconsistency risk
30 |
31 | ## Solution Implemented
32 |
33 | ### 1. Created Shared Service Layer
34 |
35 | **File**: `src/mcp_memory_service/services/memory_service.py`
36 |
37 | - **Single source of truth** for memory listing logic
38 | - Consistent pagination and filtering across all interfaces
39 | - Proper error handling and logging
40 | - Separate formatting methods for different response types
41 |
42 | ### 2. Refactored All Implementations
43 |
44 | **Updated Files:**
45 | - `src/mcp_memory_service/web/api/memories.py` - API endpoint
46 | - `src/mcp_memory_service/mcp_server.py` - MCP tool
47 | - `src/mcp_memory_service/web/api/mcp.py` - MCP API endpoint
48 |
49 | **All now use**: `MemoryService.list_memories()` for consistent behavior
50 |
51 | ## Benefits Achieved
52 |
53 | ### ✅ **Consistency**
54 | - All interfaces now use identical business logic
55 | - No more data loss or incorrect pagination
56 | - Consistent error handling
57 |
58 | ### ✅ **Maintainability**
59 | - Single place to update memory listing logic
60 | - Reduced code duplication by ~80%
61 | - Easier to add new features or fix bugs
62 |
63 | ### ✅ **Reliability**
64 | - Proper pagination with accurate counts
65 | - Correct tag and memory_type filtering
66 | - Better error handling and logging
67 |
68 | ### ✅ **Testability**
69 | - Service layer can be unit tested independently
70 | - Easier to mock and test different scenarios
71 |
72 | ## Architecture Pattern
73 |
74 | ```
75 | ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
76 | │ API Endpoint │ │ MCP Tool │ │ MCP API │
77 | │ │ │ │ │ │
78 | └─────────┬───────┘ └─────────┬───────┘ └─────────┬───────┘
79 | │ │ │
80 | └──────────────────────┼──────────────────────┘
81 | │
82 | ┌─────────────▼─────────────┐
83 | │ MemoryService │
84 | │ (Shared Business Logic) │
85 | └─────────────┬─────────────┘
86 | │
87 | ┌─────────────▼─────────────┐
88 | │ MemoryStorage │
89 | │ (Data Access Layer) │
90 | └───────────────────────────┘
91 | ```
92 |
93 | ## Best Practices Applied
94 |
95 | 1. **Single Responsibility Principle**: Service layer handles only business logic
96 | 2. **DRY (Don't Repeat Yourself)**: Eliminated code duplication
97 | 3. **Separation of Concerns**: Business logic separated from presentation logic
98 | 4. **Consistent Interface**: All consumers use the same service methods
99 | 5. **Error Handling**: Centralized error handling and logging
100 |
101 | ## Future Recommendations
102 |
103 | 1. **Apply Same Pattern**: Consider refactoring other operations (store, delete, search) to use shared services
104 | 2. **Add Validation**: Move input validation to the service layer
105 | 3. **Add Caching**: Implement caching at the service layer if needed
106 | 4. **Add Metrics**: Add performance metrics and monitoring to the service layer
107 |
108 | ## Testing Recommendations
109 |
110 | 1. **Unit Tests**: Test `MemoryService` independently
111 | 2. **Integration Tests**: Test each interface (API, MCP) with the service
112 | 3. **End-to-End Tests**: Verify consistent behavior across all interfaces
113 |
114 | This refactoring ensures that all memory listing operations behave identically regardless of the interface used, eliminating the data loss and inconsistency issues that were present in the original implementation.
115 |
116 | ## Phase 2: Complete Service Layer Refactoring
117 |
118 | ### Tools Refactoring Analysis
119 |
120 | Based on comprehensive analysis of the codebase, **8 total tools** need to be refactored to use the shared `MemoryService` pattern:
121 |
122 | #### ✅ **COMPLETED (1 tool):**
123 | 1. **`list_memories`** - ✅ **DONE** - Already uses `MemoryService.list_memories()`
124 |
125 | #### 🔄 **PENDING REFACTORING (7 tools):**
126 |
127 | ##### **Core Memory Operations (4 tools):**
128 | 2. **`store_memory`** - **HIGH PRIORITY**
129 | - **Current Issues**: Duplicated logic in 3 files
130 | - **Files**: `mcp_server.py` (lines 154-217), `web/api/memories.py` (lines 100-182), `web/api/mcp.py` (lines 247-286)
131 | - **Service Method Needed**: `MemoryService.store_memory()`
132 |
133 | 3. **`retrieve_memory`** - **HIGH PRIORITY**
134 | - **Current Issues**: Duplicated logic in 2 files
135 | - **Files**: `mcp_server.py` (lines 219-271), `web/api/mcp.py` (lines 288-315)
136 | - **Service Method Needed**: `MemoryService.retrieve_memory()`
137 |
138 | 4. **`search_by_tag`** - **HIGH PRIORITY**
139 | - **Current Issues**: Duplicated logic in 2 files
140 | - **Files**: `mcp_server.py` (lines 273-326), `web/api/mcp.py` (lines 317-370)
141 | - **Service Method Needed**: `MemoryService.search_by_tag()`
142 |
143 | 5. **`delete_memory`** - **HIGH PRIORITY**
144 | - **Current Issues**: Duplicated logic in 3 files
145 | - **Files**: `mcp_server.py` (lines 328-360), `web/api/memories.py` (lines 248-276), `web/api/mcp.py` (lines 372-380)
146 | - **Service Method Needed**: `MemoryService.delete_memory()`
147 |
148 | ##### **Advanced Search Operations (2 tools):**
149 | 6. **`search_by_time`** - **MEDIUM PRIORITY**
150 | - **Current Issues**: Duplicated logic in 2 files
151 | - **Files**: `mcp_server.py` (lines 442-516), `web/api/mcp.py` (lines 417-468)
152 | - **Service Method Needed**: `MemoryService.search_by_time()`
153 |
154 | 7. **`search_similar`** - **MEDIUM PRIORITY**
155 | - **Current Issues**: Duplicated logic in 2 files
156 | - **Files**: `mcp_server.py` (lines 518-584), `web/api/mcp.py` (lines 470-512)
157 | - **Service Method Needed**: `MemoryService.search_similar()`
158 |
159 | ##### **Health Check (1 tool):**
160 | 8. **`check_database_health`** - **LOW PRIORITY**
161 | - **Current Issues**: Duplicated logic in 2 files
162 | - **Files**: `mcp_server.py` (lines 362-394), `web/api/mcp.py` (lines 382-395)
163 | - **Service Method Needed**: `MemoryService.check_database_health()`
164 |
165 | ### Refactoring Progress Tracking
166 |
167 | | Tool | Priority | Status | Service Method | MCP Server | API Endpoint | MCP API |
168 | |------|----------|--------|----------------|------------|--------------|---------|
169 | | `list_memories` | HIGH | ✅ DONE | ✅ `list_memories()` | ✅ Refactored | ✅ Refactored | ✅ Refactored |
170 | | `store_memory` | HIGH | ✅ DONE | ✅ `store_memory()` | ✅ Refactored | ✅ Refactored | ✅ Refactored |
171 | | `retrieve_memory` | HIGH | ✅ DONE | ✅ `retrieve_memory()` | ✅ Refactored | ✅ Refactored | ✅ Refactored |
172 | | `search_by_tag` | HIGH | ✅ DONE | ✅ `search_by_tag()` | ✅ Refactored | ✅ Refactored | ✅ Refactored |
173 | | `delete_memory` | HIGH | ✅ DONE | ✅ `delete_memory()` | ✅ Refactored | ✅ Refactored | ✅ Refactored |
174 | | `search_by_time` | MEDIUM | ✅ DONE | ✅ `search_by_time()` | ✅ Refactored | ✅ Refactored | ✅ Refactored |
175 | | `search_similar` | MEDIUM | ✅ DONE | ✅ `search_similar()` | ✅ Refactored | ✅ Refactored | ✅ Refactored |
176 | | `check_database_health` | LOW | ✅ DONE | ✅ `check_database_health()` | ✅ Refactored | N/A | ✅ Refactored |
177 |
178 | ### Implementation Plan
179 |
180 | #### **Phase 2A: Core Operations (High Priority)**
181 | 1. ✅ **COMPLETED** - Create `MemoryService.store_memory()` method
182 | 2. Create `MemoryService.retrieve_memory()` method
183 | 3. Create `MemoryService.search_by_tag()` method
184 | 4. Create `MemoryService.delete_memory()` method
185 | 5. ✅ **COMPLETED** - Refactor all 3 interfaces to use new service methods
186 |
187 | #### **Phase 2A.1: `store_memory` Refactoring - COMPLETED ✅**
188 |
189 | **Service Method Created:**
190 | - ✅ `MemoryService.store_memory()` - API-based implementation
191 | - ✅ Hostname priority: Client → HTTP Header → Server
192 | - ✅ Content hash generation with metadata
193 | - ✅ Complete error handling and logging
194 | - ✅ Memory object creation and storage
195 |
196 | **Interfaces Refactored:**
197 | - ✅ **MCP Server** - Uses `MemoryService.store_memory()`
198 | - ✅ **API Endpoint** - Uses `MemoryService.store_memory()` with SSE events
199 | - ✅ **MCP API** - Uses `MemoryService.store_memory()`
200 |
201 | **Testing Completed:**
202 | - ✅ **Manual Testing** - Both user and AI tested successfully
203 | - ✅ **Sample Data Storage** - Verified with real data
204 | - ✅ **Tag and Metadata Handling** - Confirmed working
205 | - ✅ **Client Hostname Processing** - Verified automatic addition
206 | - ✅ **Content Hash Generation** - Confirmed consistency
207 | - ✅ **Memory Retrieval** - Verified stored memories can be found
208 |
209 | **Code Reduction:**
210 | - ✅ **~70% reduction** in duplicated business logic
211 | - ✅ **Single source of truth** for memory storage
212 | - ✅ **Consistent behavior** across all interfaces
213 |
214 | #### **Phase 2A.2: `retrieve_memory` Refactoring - COMPLETED ✅**
215 |
216 | **Service Method Created:**
217 | - ✅ `MemoryService.retrieve_memory()` - API-based implementation (`/api/search`)
218 | - ✅ Uses exact API logic as source of truth
219 | - ✅ Handles semantic search, similarity filtering, processing time
220 | - ✅ Returns consistent response format with `SearchResult` structure
221 |
222 | **Interfaces Refactored:**
223 | - ✅ **API Endpoint** - Refactored to use service method (eliminated duplication)
224 | - ✅ **MCP Server** - Refactored to use service method
225 | - ✅ **MCP API** - Refactored to use service method
226 |
227 | **Testing Completed:**
228 | - ✅ **Exact Matches** - Perfect similarity scores (1.0) for identical content
229 | - ✅ **Partial Matches** - Reasonable similarity scores (0.121, 0.118, 0.135)
230 | - ✅ **Similarity Filtering** - Threshold filtering working correctly
231 | - ✅ **Processing Time** - Timing metrics included (~13ms)
232 | - ✅ **Response Format** - Consistent across all interfaces
233 | - ✅ **Manual Testing** - User tested with real queries and thresholds
234 | - ✅ **Production Ready** - All interfaces working correctly in live environment
235 |
236 | **Key Features:**
237 | - ✅ **Semantic Search**: Uses vector embeddings for similarity
238 | - ✅ **Similarity Filtering**: Post-processing threshold filtering
239 | - ✅ **Processing Time**: Includes timing metrics
240 | - ✅ **Relevance Reasoning**: Explains why results were included
241 | - ✅ **SSE Events**: Maintains real-time event broadcasting
242 |
243 | **Code Reduction:**
244 | - ✅ **~60% reduction** in duplicated search logic
245 | - ✅ **Single source of truth** for memory retrieval
246 | - ✅ **Consistent behavior** across all interfaces
247 |
248 | #### **Phase 2A.3: `search_by_tag` Refactoring - COMPLETED ✅**
249 |
250 | **Service Method Created:**
251 | - ✅ `MemoryService.search_by_tag()` - API-based implementation (`/api/search/by-tag`)
252 | - ✅ Uses exact API logic as source of truth
253 | - ✅ Handles tag filtering with AND/OR operations (match_all parameter)
254 | - ✅ Returns consistent response format with `SearchResult` structure
255 | - ✅ Processing time metrics and proper error handling
256 |
257 | **Interfaces Refactored:**
258 | - ✅ **API Endpoint** - Refactored to use service method (eliminated duplication)
259 | - ✅ **MCP Server** - Refactored to use service method with parameter conversion
260 | - ✅ **MCP API** - Refactored to use service method while preserving string parsing
261 |
262 | **Testing Completed:**
263 | - ✅ **Tag Matching** - Both ANY and ALL tag matching modes working correctly
264 | - ✅ **Parameter Conversion** - Proper handling of operation string vs match_all boolean
265 | - ✅ **Response Format** - Consistent SearchResult format across all interfaces
266 | - ✅ **Error Handling** - Validation errors properly handled and converted
267 | - ✅ **Manual Testing** - User tested with real tag queries and confirmed working
268 | - ✅ **Production Ready** - All interfaces working correctly in live environment
269 |
270 | **Key Features:**
271 | - ✅ **Tag Search**: Finds memories containing specified tags
272 | - ✅ **AND/OR Operations**: Supports both any tag match and all tags match
273 | - ✅ **Processing Time**: Includes timing metrics for performance monitoring
274 | - ✅ **Relevance Reasoning**: Explains which tags matched for transparency
275 | - ✅ **SSE Events**: Maintains real-time event broadcasting
276 |
277 | **Code Reduction:**
278 | - ✅ **~65% reduction** in duplicated tag search logic
279 | - ✅ **Single source of truth** for tag-based memory search
280 | - ✅ **Consistent behavior** across all interfaces
281 |
282 | #### **Phase 2A.4: `delete_memory` Refactoring - COMPLETED ✅**
283 |
284 | **Service Method Created:**
285 | - ✅ `MemoryService.delete_memory()` - API-based implementation (`/api/memories/{content_hash}`)
286 | - ✅ Uses exact API logic as source of truth
287 | - ✅ Handles content hash validation and storage layer deletion
288 | - ✅ Returns consistent response format with success/message/content_hash
289 | - ✅ Comprehensive error handling and logging
290 |
291 | **Interfaces Refactored:**
292 | - ✅ **API Endpoint** - Refactored to use service method (eliminated duplication)
293 | - ✅ **MCP Server** - Refactored to use service method
294 | - ✅ **MCP API** - Refactored to use service method
295 |
296 | **Testing Completed:**
297 | - ✅ **Service Method Testing** - Direct testing of MemoryService.delete_memory()
298 | - ✅ **Storage Integration** - Verified memory creation and deletion workflow
299 | - ✅ **Manual Testing** - User tested with real memory hashes and confirmed working
300 | - ✅ **Production Ready** - All interfaces working correctly in live environment
301 |
302 | **Key Features:**
303 | - ✅ **Content Hash Validation**: Validates input parameters before processing
304 | - ✅ **Storage Integration**: Uses storage layer delete() method for consistency
305 | - ✅ **Error Handling**: Comprehensive error handling with detailed messages
306 | - ✅ **Response Consistency**: Uniform response format across all interfaces
307 | - ✅ **SSE Events**: Maintains real-time event broadcasting for web dashboard
308 |
309 | **Code Reduction:**
310 | - ✅ **~70% reduction** in duplicated deletion logic
311 | - ✅ **Single source of truth** for memory deletion
312 | - ✅ **Consistent behavior** across all interfaces
313 |
314 | #### **Phase 2B: Advanced Search (Medium Priority)**
315 | 6. Create `MemoryService.search_by_time()` method
316 | 7. Create `MemoryService.search_similar()` method
317 | 8. Refactor MCP server and MCP API to use new service methods
318 |
319 | #### **Phase 2C: Health Check (Low Priority) - COMPLETED ✅**
320 |
321 | **Service Method Created:**
322 | - ✅ `MemoryService.check_database_health()` - MCP Server-based implementation
323 | - ✅ Handles both async and sync storage `get_stats()` methods
324 | - ✅ Maps storage backend fields to consistent health check format
325 | - ✅ Includes comprehensive statistics: memories, tags, storage size, embedding info
326 | - ✅ Complete error handling with detailed error responses
327 |
328 | **Interfaces Refactored:**
329 | - ✅ **MCP Server** - Uses `MemoryService.check_database_health()`
330 | - ✅ **MCP API** - Uses `MemoryService.check_database_health()`
331 |
332 | **Key Features:**
333 | - ✅ **Field Mapping**: Handles variations between storage backends (`unique_tags` → `total_tags`, `database_size_mb` → formatted size)
334 | - ✅ **Async/Sync Compatibility**: Detects and handles both async and sync `get_stats()` methods
335 | - ✅ **Comprehensive Statistics**: Includes embedding model info, storage size, and backend details
336 | - ✅ **Error Handling**: Proper error responses for storage backend failures
337 | - ✅ **Consistent Format**: Unified health check response across all interfaces
338 |
339 | **Testing Completed:**
340 | - ✅ **Field Mapping Fix** - Resolved user-reported issues with `total_tags`, `storage_size`, and `timestamp` fields
341 | - ✅ **Storage Backend Integration** - Verified compatibility with SQLite-Vec storage
342 | - ✅ **Manual Testing** - User confirmed health check now returns proper field values
343 | - ✅ **Production Ready** - All interfaces working correctly with enhanced statistics
344 |
345 | **Code Reduction:**
346 | - ✅ **~60% reduction** in duplicated health check logic
347 | - ✅ **Single source of truth** for database health monitoring
348 | - ✅ **Consistent behavior** across all interfaces
349 |
350 | ### Expected Benefits
351 |
352 | - **Consistency**: All 8 tools will have identical behavior across all interfaces
353 | - **Maintainability**: Single source of truth for all memory operations
354 | - **Code Reduction**: ~70% reduction in duplicated business logic
355 | - **Reliability**: Centralized error handling and validation
356 | - **Testability**: Service layer can be unit tested independently
357 |
358 | ### Success Metrics
359 |
360 | - ✅ **Zero Code Duplication**: No business logic duplicated across interfaces
361 | - ✅ **100% Consistency**: All tools behave identically regardless of interface
362 | - ✅ **Single Source of Truth**: All operations go through `MemoryService`
363 | - ✅ **Comprehensive Testing**: Service layer fully tested independently
364 |
```
--------------------------------------------------------------------------------
/src/mcp_memory_service/consolidation/scheduler.py:
--------------------------------------------------------------------------------
```python
1 | # Copyright 2024 Heinrich Krupp
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """APScheduler integration for autonomous consolidation operations."""
16 |
17 | import asyncio
18 | import logging
19 | from typing import Dict, Any, Optional, Callable, Awaitable
20 | from datetime import datetime, timedelta
21 |
22 | try:
23 | from apscheduler.schedulers.asyncio import AsyncIOScheduler
24 | from apscheduler.triggers.cron import CronTrigger
25 | from apscheduler.triggers.interval import IntervalTrigger
26 | from apscheduler.jobstores.memory import MemoryJobStore
27 | from apscheduler.executors.asyncio import AsyncIOExecutor
28 | from apscheduler.events import EVENT_JOB_EXECUTED, EVENT_JOB_ERROR
29 | APSCHEDULER_AVAILABLE = True
30 | except ImportError:
31 | APSCHEDULER_AVAILABLE = False
32 |
33 | from .consolidator import DreamInspiredConsolidator
34 | from .base import ConsolidationConfig
35 |
36 | class ConsolidationScheduler:
37 | """
38 | Scheduler for autonomous consolidation operations.
39 |
40 | Integrates with APScheduler to run consolidation operations at specified intervals
41 | based on time horizons (daily, weekly, monthly, quarterly, yearly).
42 | """
43 |
44 | def __init__(
45 | self,
46 | consolidator: DreamInspiredConsolidator,
47 | schedule_config: Dict[str, str],
48 | enabled: bool = True
49 | ):
50 | self.consolidator = consolidator
51 | self.schedule_config = schedule_config
52 | self.enabled = enabled
53 | self.logger = logging.getLogger(__name__)
54 |
55 | # Job execution tracking
56 | self.job_history = []
57 | self.last_execution_times = {}
58 | self.execution_stats = {
59 | 'total_jobs': 0,
60 | 'successful_jobs': 0,
61 | 'failed_jobs': 0
62 | }
63 |
64 | # Initialize scheduler if APScheduler is available
65 | if APSCHEDULER_AVAILABLE and enabled:
66 | self.scheduler = AsyncIOScheduler(
67 | jobstores={'default': MemoryJobStore()},
68 | executors={'default': AsyncIOExecutor()},
69 | job_defaults={
70 | 'coalesce': True, # Combine multiple pending executions
71 | 'max_instances': 1, # Only one instance of each job at a time
72 | 'misfire_grace_time': 3600 # 1 hour grace period for missed jobs
73 | }
74 | )
75 |
76 | # Add event listeners
77 | self.scheduler.add_listener(self._job_executed_listener, EVENT_JOB_EXECUTED)
78 | self.scheduler.add_listener(self._job_error_listener, EVENT_JOB_ERROR)
79 | else:
80 | self.scheduler = None
81 | if not APSCHEDULER_AVAILABLE:
82 | self.logger.warning("APScheduler not available - consolidation scheduling disabled")
83 | elif not enabled:
84 | self.logger.info("Consolidation scheduling disabled by configuration")
85 |
86 | async def start(self) -> bool:
87 | """Start the consolidation scheduler."""
88 | if not self.scheduler:
89 | return False
90 |
91 | try:
92 | # Add consolidation jobs based on configuration
93 | self._schedule_consolidation_jobs()
94 |
95 | # Start the scheduler
96 | self.scheduler.start()
97 | self.logger.info("Consolidation scheduler started successfully")
98 |
99 | # Log scheduled jobs
100 | jobs = self.scheduler.get_jobs()
101 | for job in jobs:
102 | self.logger.info(f"Scheduled job: {job.id} - next run: {job.next_run_time}")
103 |
104 | return True
105 |
106 | except Exception as e:
107 | self.logger.error(f"Failed to start consolidation scheduler: {e}")
108 | return False
109 |
110 | async def stop(self) -> bool:
111 | """Stop the consolidation scheduler."""
112 | if not self.scheduler:
113 | return True
114 |
115 | try:
116 | self.scheduler.shutdown(wait=True)
117 | self.logger.info("Consolidation scheduler stopped")
118 | return True
119 | except Exception as e:
120 | self.logger.error(f"Error stopping consolidation scheduler: {e}")
121 | return False
122 |
123 | def _schedule_consolidation_jobs(self):
124 | """Schedule consolidation jobs based on configuration."""
125 | time_horizons = ['daily', 'weekly', 'monthly', 'quarterly', 'yearly']
126 |
127 | for horizon in time_horizons:
128 | schedule_spec = self.schedule_config.get(horizon, 'disabled')
129 |
130 | if schedule_spec == 'disabled':
131 | self.logger.debug(f"Consolidation for {horizon} horizon is disabled")
132 | continue
133 |
134 | try:
135 | trigger = self._create_trigger(horizon, schedule_spec)
136 | if trigger:
137 | job_id = f"consolidation_{horizon}"
138 | self.scheduler.add_job(
139 | func=self._run_consolidation_job,
140 | trigger=trigger,
141 | args=[horizon],
142 | id=job_id,
143 | name=f"Consolidation - {horizon.title()}",
144 | replace_existing=True
145 | )
146 | self.logger.info(f"Scheduled {horizon} consolidation: {schedule_spec}")
147 |
148 | except Exception as e:
149 | self.logger.error(f"Error scheduling {horizon} consolidation: {e}")
150 |
151 | def _create_trigger(self, horizon: str, schedule_spec: str):
152 | """Create APScheduler trigger from schedule specification."""
153 | try:
154 | if horizon == 'daily':
155 | # Daily format: "HH:MM" (e.g., "02:00")
156 | hour, minute = map(int, schedule_spec.split(':'))
157 | return CronTrigger(hour=hour, minute=minute)
158 |
159 | elif horizon == 'weekly':
160 | # Weekly format: "DAY HH:MM" (e.g., "SUN 03:00")
161 | day_time = schedule_spec.split(' ')
162 | if len(day_time) != 2:
163 | raise ValueError(f"Invalid weekly schedule format: {schedule_spec}")
164 |
165 | day_map = {
166 | 'MON': 0, 'TUE': 1, 'WED': 2, 'THU': 3,
167 | 'FRI': 4, 'SAT': 5, 'SUN': 6
168 | }
169 |
170 | day = day_map.get(day_time[0].upper())
171 | if day is None:
172 | raise ValueError(f"Invalid day: {day_time[0]}")
173 |
174 | hour, minute = map(int, day_time[1].split(':'))
175 | return CronTrigger(day_of_week=day, hour=hour, minute=minute)
176 |
177 | elif horizon == 'monthly':
178 | # Monthly format: "DD HH:MM" (e.g., "01 04:00")
179 | day_time = schedule_spec.split(' ')
180 | if len(day_time) != 2:
181 | raise ValueError(f"Invalid monthly schedule format: {schedule_spec}")
182 |
183 | day = int(day_time[0])
184 | hour, minute = map(int, day_time[1].split(':'))
185 | return CronTrigger(day=day, hour=hour, minute=minute)
186 |
187 | elif horizon == 'quarterly':
188 | # Quarterly format: "MM-DD HH:MM" (e.g., "01-01 05:00")
189 | # Run on the first day of quarters (Jan, Apr, Jul, Oct)
190 | parts = schedule_spec.split(' ')
191 | if len(parts) != 2:
192 | raise ValueError(f"Invalid quarterly schedule format: {schedule_spec}")
193 |
194 | month_day = parts[0].split('-')
195 | if len(month_day) != 2:
196 | raise ValueError(f"Invalid quarterly date format: {parts[0]}")
197 |
198 | day = int(month_day[1])
199 | hour, minute = map(int, parts[1].split(':'))
200 |
201 | # Quarters: Jan(1), Apr(4), Jul(7), Oct(10)
202 | return CronTrigger(month='1,4,7,10', day=day, hour=hour, minute=minute)
203 |
204 | elif horizon == 'yearly':
205 | # Yearly format: "MM-DD HH:MM" (e.g., "01-01 06:00")
206 | parts = schedule_spec.split(' ')
207 | if len(parts) != 2:
208 | raise ValueError(f"Invalid yearly schedule format: {schedule_spec}")
209 |
210 | month_day = parts[0].split('-')
211 | if len(month_day) != 2:
212 | raise ValueError(f"Invalid yearly date format: {parts[0]}")
213 |
214 | month = int(month_day[0])
215 | day = int(month_day[1])
216 | hour, minute = map(int, parts[1].split(':'))
217 |
218 | return CronTrigger(month=month, day=day, hour=hour, minute=minute)
219 |
220 | else:
221 | self.logger.error(f"Unknown time horizon: {horizon}")
222 | return None
223 |
224 | except Exception as e:
225 | self.logger.error(f"Error creating trigger for {horizon} with spec '{schedule_spec}': {e}")
226 | return None
227 |
228 | async def _run_consolidation_job(self, time_horizon: str):
229 | """Execute a consolidation job for the specified time horizon."""
230 | job_start_time = datetime.now()
231 | self.logger.info(f"Starting scheduled {time_horizon} consolidation")
232 |
233 | try:
234 | # Run the consolidation
235 | report = await self.consolidator.consolidate(time_horizon)
236 |
237 | # Record successful execution
238 | self.execution_stats['successful_jobs'] += 1
239 | self.last_execution_times[time_horizon] = job_start_time
240 |
241 | # Add to job history
242 | job_record = {
243 | 'time_horizon': time_horizon,
244 | 'start_time': job_start_time,
245 | 'end_time': datetime.now(),
246 | 'status': 'success',
247 | 'memories_processed': report.memories_processed,
248 | 'associations_discovered': report.associations_discovered,
249 | 'clusters_created': report.clusters_created,
250 | 'memories_compressed': report.memories_compressed,
251 | 'memories_archived': report.memories_archived,
252 | 'errors': report.errors
253 | }
254 |
255 | self._add_job_to_history(job_record)
256 |
257 | # Log success
258 | duration = (job_record['end_time'] - job_record['start_time']).total_seconds()
259 | self.logger.info(
260 | f"Completed {time_horizon} consolidation successfully in {duration:.2f}s: "
261 | f"{report.memories_processed} memories processed, "
262 | f"{report.associations_discovered} associations, "
263 | f"{report.clusters_created} clusters, "
264 | f"{report.memories_compressed} compressed, "
265 | f"{report.memories_archived} archived"
266 | )
267 |
268 | except Exception as e:
269 | # Record failed execution
270 | self.execution_stats['failed_jobs'] += 1
271 |
272 | job_record = {
273 | 'time_horizon': time_horizon,
274 | 'start_time': job_start_time,
275 | 'end_time': datetime.now(),
276 | 'status': 'failed',
277 | 'error': str(e),
278 | 'memories_processed': 0,
279 | 'associations_discovered': 0,
280 | 'clusters_created': 0,
281 | 'memories_compressed': 0,
282 | 'memories_archived': 0,
283 | 'errors': [str(e)]
284 | }
285 |
286 | self._add_job_to_history(job_record)
287 |
288 | self.logger.error(f"Failed {time_horizon} consolidation: {e}")
289 | raise
290 |
291 | def _add_job_to_history(self, job_record: Dict[str, Any]):
292 | """Add job record to history with size limit."""
293 | self.job_history.append(job_record)
294 |
295 | # Keep only last 100 job records
296 | if len(self.job_history) > 100:
297 | self.job_history = self.job_history[-100:]
298 |
299 | def _job_executed_listener(self, event):
300 | """Handle job execution events."""
301 | self.execution_stats['total_jobs'] += 1
302 | self.logger.debug(f"Job executed: {event.job_id}")
303 |
304 | def _job_error_listener(self, event):
305 | """Handle job error events."""
306 | self.logger.error(f"Job error: {event.job_id} - {event.exception}")
307 |
308 | async def trigger_consolidation(self, time_horizon: str, immediate: bool = True) -> bool:
309 | """Manually trigger a consolidation job."""
310 | if not self.scheduler:
311 | self.logger.error("Scheduler not available")
312 | return False
313 |
314 | try:
315 | if immediate:
316 | # Run immediately
317 | await self._run_consolidation_job(time_horizon)
318 | return True
319 | else:
320 | # Schedule to run in 1 minute
321 | job_id = f"manual_consolidation_{time_horizon}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
322 | trigger = IntervalTrigger(seconds=60) # Run once after 60 seconds
323 |
324 | self.scheduler.add_job(
325 | func=self._run_consolidation_job,
326 | trigger=trigger,
327 | args=[time_horizon],
328 | id=job_id,
329 | name=f"Manual Consolidation - {time_horizon.title()}",
330 | max_instances=1
331 | )
332 |
333 | self.logger.info(f"Scheduled manual {time_horizon} consolidation")
334 | return True
335 |
336 | except Exception as e:
337 | self.logger.error(f"Error triggering {time_horizon} consolidation: {e}")
338 | return False
339 |
340 | async def get_scheduler_status(self) -> Dict[str, Any]:
341 | """Get scheduler status and job information."""
342 | if not self.scheduler:
343 | return {
344 | 'enabled': False,
345 | 'reason': 'APScheduler not available or disabled'
346 | }
347 |
348 | jobs = self.scheduler.get_jobs()
349 | job_info = []
350 |
351 | for job in jobs:
352 | job_info.append({
353 | 'id': job.id,
354 | 'name': job.name,
355 | 'next_run_time': job.next_run_time.isoformat() if job.next_run_time else None,
356 | 'trigger': str(job.trigger)
357 | })
358 |
359 | return {
360 | 'enabled': True,
361 | 'running': self.scheduler.running,
362 | 'jobs': job_info,
363 | 'execution_stats': self.execution_stats.copy(),
364 | 'last_execution_times': {
365 | horizon: time.isoformat() for horizon, time in self.last_execution_times.items()
366 | },
367 | 'recent_jobs': self.job_history[-10:] # Last 10 jobs
368 | }
369 |
370 | async def update_schedule(self, new_schedule_config: Dict[str, str]) -> bool:
371 | """Update the consolidation schedule."""
372 | if not self.scheduler:
373 | return False
374 |
375 | try:
376 | # Remove existing consolidation jobs
377 | job_ids = [f"consolidation_{horizon}" for horizon in ['daily', 'weekly', 'monthly', 'quarterly', 'yearly']]
378 |
379 | for job_id in job_ids:
380 | if self.scheduler.get_job(job_id):
381 | self.scheduler.remove_job(job_id)
382 |
383 | # Update configuration
384 | self.schedule_config = new_schedule_config
385 |
386 | # Re-schedule jobs
387 | self._schedule_consolidation_jobs()
388 |
389 | self.logger.info("Consolidation schedule updated successfully")
390 | return True
391 |
392 | except Exception as e:
393 | self.logger.error(f"Error updating consolidation schedule: {e}")
394 | return False
395 |
396 | async def pause_consolidation(self, time_horizon: Optional[str] = None) -> bool:
397 | """Pause consolidation jobs (all or specific horizon)."""
398 | if not self.scheduler:
399 | return False
400 |
401 | try:
402 | if time_horizon:
403 | job_id = f"consolidation_{time_horizon}"
404 | job = self.scheduler.get_job(job_id)
405 | if job:
406 | self.scheduler.pause_job(job_id)
407 | self.logger.info(f"Paused {time_horizon} consolidation")
408 | else:
409 | self.logger.warning(f"No job found for {time_horizon} consolidation")
410 | else:
411 | # Pause all consolidation jobs
412 | jobs = self.scheduler.get_jobs()
413 | for job in jobs:
414 | if job.id.startswith('consolidation_'):
415 | self.scheduler.pause_job(job.id)
416 |
417 | self.logger.info("Paused all consolidation jobs")
418 |
419 | return True
420 |
421 | except Exception as e:
422 | self.logger.error(f"Error pausing consolidation: {e}")
423 | return False
424 |
425 | async def resume_consolidation(self, time_horizon: Optional[str] = None) -> bool:
426 | """Resume consolidation jobs (all or specific horizon)."""
427 | if not self.scheduler:
428 | return False
429 |
430 | try:
431 | if time_horizon:
432 | job_id = f"consolidation_{time_horizon}"
433 | job = self.scheduler.get_job(job_id)
434 | if job:
435 | self.scheduler.resume_job(job_id)
436 | self.logger.info(f"Resumed {time_horizon} consolidation")
437 | else:
438 | self.logger.warning(f"No job found for {time_horizon} consolidation")
439 | else:
440 | # Resume all consolidation jobs
441 | jobs = self.scheduler.get_jobs()
442 | for job in jobs:
443 | if job.id.startswith('consolidation_'):
444 | self.scheduler.resume_job(job.id)
445 |
446 | self.logger.info("Resumed all consolidation jobs")
447 |
448 | return True
449 |
450 | except Exception as e:
451 | self.logger.error(f"Error resuming consolidation: {e}")
452 | return False
```
--------------------------------------------------------------------------------
/tests/consolidation/test_consolidator.py:
--------------------------------------------------------------------------------
```python
1 | """Integration tests for the main dream-inspired consolidator."""
2 |
3 | import pytest
4 | from datetime import datetime, timedelta
5 | from unittest.mock import AsyncMock, MagicMock
6 |
7 | from mcp_memory_service.consolidation.consolidator import DreamInspiredConsolidator
8 | from mcp_memory_service.consolidation.base import ConsolidationReport
9 | from mcp_memory_service.models.memory import Memory
10 |
11 |
12 | @pytest.mark.integration
13 | class TestDreamInspiredConsolidator:
14 | """Test the main consolidation orchestrator."""
15 |
16 | @pytest.fixture
17 | def consolidator(self, mock_storage, consolidation_config):
18 | return DreamInspiredConsolidator(mock_storage, consolidation_config)
19 |
20 | @pytest.mark.asyncio
21 | async def test_basic_consolidation_pipeline(self, consolidator, mock_storage):
22 | """Test the complete consolidation pipeline."""
23 | report = await consolidator.consolidate("weekly")
24 |
25 | assert isinstance(report, ConsolidationReport)
26 | assert report.time_horizon == "weekly"
27 | assert isinstance(report.start_time, datetime)
28 | assert isinstance(report.end_time, datetime)
29 | assert report.end_time >= report.start_time
30 | assert report.memories_processed >= 0
31 | assert report.associations_discovered >= 0
32 | assert report.clusters_created >= 0
33 | assert report.memories_compressed >= 0
34 | assert report.memories_archived >= 0
35 | assert isinstance(report.errors, list)
36 | assert isinstance(report.performance_metrics, dict)
37 |
38 | @pytest.mark.asyncio
39 | async def test_daily_consolidation(self, consolidator):
40 | """Test daily consolidation (light processing)."""
41 | report = await consolidator.consolidate("daily")
42 |
43 | assert report.time_horizon == "daily"
44 | # Daily consolidation should be lighter - less intensive operations
45 | assert isinstance(report, ConsolidationReport)
46 |
47 | @pytest.mark.asyncio
48 | async def test_weekly_consolidation(self, consolidator):
49 | """Test weekly consolidation (includes associations)."""
50 | report = await consolidator.consolidate("weekly")
51 |
52 | assert report.time_horizon == "weekly"
53 | # Weekly should include association discovery
54 | assert isinstance(report, ConsolidationReport)
55 |
56 | @pytest.mark.asyncio
57 | async def test_monthly_consolidation(self, consolidator):
58 | """Test monthly consolidation (includes forgetting)."""
59 | report = await consolidator.consolidate("monthly")
60 |
61 | assert report.time_horizon == "monthly"
62 | # Monthly should include more comprehensive processing
63 | assert isinstance(report, ConsolidationReport)
64 |
65 | @pytest.mark.asyncio
66 | async def test_quarterly_consolidation(self, consolidator):
67 | """Test quarterly consolidation (deep processing)."""
68 | report = await consolidator.consolidate("quarterly")
69 |
70 | assert report.time_horizon == "quarterly"
71 | # Quarterly should include all processing steps
72 | assert isinstance(report, ConsolidationReport)
73 |
74 | @pytest.mark.asyncio
75 | async def test_yearly_consolidation(self, consolidator):
76 | """Test yearly consolidation (full processing)."""
77 | report = await consolidator.consolidate("yearly")
78 |
79 | assert report.time_horizon == "yearly"
80 | # Yearly should include comprehensive forgetting
81 | assert isinstance(report, ConsolidationReport)
82 |
83 | @pytest.mark.asyncio
84 | async def test_invalid_time_horizon(self, consolidator):
85 | """Test handling of invalid time horizon."""
86 | from mcp_memory_service.consolidation.base import ConsolidationError
87 | with pytest.raises(ConsolidationError):
88 | await consolidator.consolidate("invalid_horizon")
89 |
90 | @pytest.mark.asyncio
91 | async def test_empty_memory_set(self, consolidation_config):
92 | """Test consolidation with empty memory set."""
93 | # Create storage with no memories
94 | empty_storage = AsyncMock()
95 | empty_storage.get_all_memories.return_value = []
96 | empty_storage.get_memories_by_time_range.return_value = []
97 | empty_storage.get_memory_connections.return_value = {}
98 | empty_storage.get_access_patterns.return_value = {}
99 |
100 | consolidator = DreamInspiredConsolidator(empty_storage, consolidation_config)
101 |
102 | report = await consolidator.consolidate("weekly")
103 |
104 | assert report.memories_processed == 0
105 | assert report.associations_discovered == 0
106 | assert report.clusters_created == 0
107 | assert report.memories_compressed == 0
108 | assert report.memories_archived == 0
109 |
110 | @pytest.mark.asyncio
111 | async def test_memories_by_time_range_retrieval(self, consolidator, mock_storage):
112 | """Test retrieval of memories by time range for daily processing."""
113 | # Mock the time range method to return specific memories
114 | recent_memories = mock_storage.memories.copy()
115 | mock_storage.get_memories_by_time_range = AsyncMock(return_value=list(recent_memories.values())[:3])
116 |
117 | report = await consolidator.consolidate("daily")
118 |
119 | # Should have called the time range method for daily processing
120 | mock_storage.get_memories_by_time_range.assert_called_once()
121 | assert report.memories_processed >= 0
122 |
123 | @pytest.mark.asyncio
124 | async def test_association_storage(self, consolidator, mock_storage):
125 | """Test that discovered associations are stored as memories."""
126 | original_memory_count = len(mock_storage.memories)
127 |
128 | # Run consolidation that should discover associations
129 | report = await consolidator.consolidate("weekly")
130 |
131 | # Check if new association memories were added
132 | current_memory_count = len(mock_storage.memories)
133 |
134 | # May or may not find associations depending on similarity
135 | # Just ensure no errors occurred
136 | assert isinstance(report, ConsolidationReport)
137 | assert current_memory_count >= original_memory_count
138 |
139 | @pytest.mark.asyncio
140 | async def test_health_check(self, consolidator):
141 | """Test consolidation system health check."""
142 | health = await consolidator.health_check()
143 |
144 | assert isinstance(health, dict)
145 | assert "status" in health
146 | assert "timestamp" in health
147 | assert "components" in health
148 | assert "statistics" in health
149 |
150 | # Check component health
151 | expected_components = [
152 | "decay_calculator",
153 | "association_engine",
154 | "clustering_engine",
155 | "compression_engine",
156 | "forgetting_engine"
157 | ]
158 |
159 | for component in expected_components:
160 | assert component in health["components"]
161 | assert "status" in health["components"][component]
162 |
163 | @pytest.mark.asyncio
164 | async def test_consolidation_recommendations(self, consolidator):
165 | """Test consolidation recommendations."""
166 | recommendations = await consolidator.get_consolidation_recommendations("weekly")
167 |
168 | assert isinstance(recommendations, dict)
169 | assert "recommendation" in recommendations
170 | assert "memory_count" in recommendations
171 |
172 | # Check recommendation types
173 | valid_recommendations = ["no_action", "consolidation_beneficial", "optional", "error"]
174 | assert recommendations["recommendation"] in valid_recommendations
175 |
176 | if recommendations["recommendation"] != "error":
177 | assert "reasons" in recommendations
178 | assert isinstance(recommendations["reasons"], list)
179 |
180 | @pytest.mark.asyncio
181 | async def test_performance_metrics(self, consolidator):
182 | """Test performance metrics collection."""
183 | report = await consolidator.consolidate("daily")
184 |
185 | assert "performance_metrics" in report.__dict__
186 | metrics = report.performance_metrics
187 |
188 | assert "duration_seconds" in metrics
189 | assert "memories_per_second" in metrics
190 | assert "success" in metrics
191 |
192 | assert isinstance(metrics["duration_seconds"], float)
193 | assert metrics["duration_seconds"] >= 0
194 | assert isinstance(metrics["memories_per_second"], (int, float))
195 | assert isinstance(metrics["success"], bool)
196 |
197 | @pytest.mark.asyncio
198 | async def test_consolidation_statistics_tracking(self, consolidator):
199 | """Test that consolidation statistics are tracked."""
200 | initial_stats = consolidator.consolidation_stats.copy()
201 |
202 | # Run consolidation
203 | await consolidator.consolidate("weekly")
204 |
205 | # Check that stats were updated
206 | assert consolidator.consolidation_stats["total_runs"] == initial_stats["total_runs"] + 1
207 |
208 | # Check other stats (may or may not be incremented depending on processing)
209 | for key in ["successful_runs", "total_memories_processed", "total_associations_created"]:
210 | assert consolidator.consolidation_stats[key] >= initial_stats[key]
211 |
212 | @pytest.mark.asyncio
213 | async def test_error_handling_in_pipeline(self, consolidation_config):
214 | """Test error handling in the consolidation pipeline."""
215 | # Create storage that raises errors
216 | error_storage = AsyncMock()
217 | error_storage.get_all_memories.side_effect = Exception("Storage error")
218 | error_storage.get_memories_by_time_range.side_effect = Exception("Storage error")
219 |
220 | consolidator = DreamInspiredConsolidator(error_storage, consolidation_config)
221 |
222 | report = await consolidator.consolidate("weekly")
223 |
224 | # Should handle errors gracefully
225 | assert len(report.errors) > 0
226 | assert report.performance_metrics["success"] is False
227 |
228 | @pytest.mark.asyncio
229 | async def test_component_integration(self, consolidator, mock_storage):
230 | """Test integration between different consolidation components."""
231 | # Ensure we have enough memories for meaningful processing
232 | if len(mock_storage.memories) < 5:
233 | # Add more memories for testing
234 | base_time = datetime.now().timestamp()
235 | for i in range(10):
236 | memory = Memory(
237 | content=f"Integration test memory {i} with content",
238 | content_hash=f"integration_{i}",
239 | tags=["integration", "test"],
240 | embedding=[0.1 + i*0.01] * 320,
241 | created_at=base_time - (i * 3600)
242 | )
243 | mock_storage.memories[memory.content_hash] = memory
244 |
245 | # Run full consolidation
246 | report = await consolidator.consolidate("monthly")
247 |
248 | # Verify that components worked together
249 | assert report.memories_processed > 0
250 |
251 | # Check that the pipeline completed successfully
252 | assert report.performance_metrics["success"] is True
253 |
254 | @pytest.mark.asyncio
255 | async def test_time_horizon_specific_processing(self, consolidator):
256 | """Test that different time horizons trigger appropriate processing."""
257 | # Test that weekly includes associations but not intensive forgetting
258 | weekly_report = await consolidator.consolidate("weekly")
259 |
260 | # Test that monthly includes forgetting
261 | monthly_report = await consolidator.consolidate("monthly")
262 |
263 | # Both should complete successfully
264 | assert weekly_report.performance_metrics["success"] is True
265 | assert monthly_report.performance_metrics["success"] is True
266 |
267 | # Monthly might have more archived memories (if forgetting triggered)
268 | # But this depends on the actual memory state, so just verify structure
269 | assert isinstance(weekly_report.memories_archived, int)
270 | assert isinstance(monthly_report.memories_archived, int)
271 |
272 | @pytest.mark.asyncio
273 | async def test_concurrent_consolidation_prevention(self, consolidator):
274 | """Test that the system handles concurrent consolidation requests appropriately."""
275 | # Start two consolidations concurrently
276 | import asyncio
277 |
278 | task1 = asyncio.create_task(consolidator.consolidate("daily"))
279 | task2 = asyncio.create_task(consolidator.consolidate("weekly"))
280 |
281 | # Both should complete (the system should handle concurrency)
282 | report1, report2 = await asyncio.gather(task1, task2)
283 |
284 | assert isinstance(report1, ConsolidationReport)
285 | assert isinstance(report2, ConsolidationReport)
286 | assert report1.time_horizon == "daily"
287 | assert report2.time_horizon == "weekly"
288 |
289 | @pytest.mark.asyncio
290 | async def test_memory_metadata_updates(self, consolidator, mock_storage):
291 | """Test that memory metadata is updated during consolidation."""
292 | original_memories = list(mock_storage.memories.values())
293 |
294 | # Run consolidation
295 | await consolidator.consolidate("weekly")
296 |
297 | # Check that memories exist (update_memory would have been called internally)
298 | # Since the mock doesn't track calls, we just verify the process completed
299 | current_memories = list(mock_storage.memories.values())
300 | assert len(current_memories) >= len(original_memories)
301 |
302 | @pytest.mark.asyncio
303 | async def test_large_memory_set_performance(self, consolidation_config, mock_large_storage):
304 | """Test performance with larger memory sets."""
305 | consolidator = DreamInspiredConsolidator(mock_large_storage, consolidation_config)
306 |
307 | start_time = datetime.now()
308 | report = await consolidator.consolidate("weekly")
309 | end_time = datetime.now()
310 |
311 | duration = (end_time - start_time).total_seconds()
312 |
313 | # Should complete within reasonable time (adjust threshold as needed)
314 | assert duration < 30 # 30 seconds for 100 memories
315 | assert report.memories_processed > 0
316 | assert report.performance_metrics["success"] is True
317 |
318 | # Performance should be reasonable
319 | if report.memories_processed > 0:
320 | memories_per_second = report.memories_processed / duration
321 | assert memories_per_second > 1 # At least 1 memory per second
322 |
323 | @pytest.mark.asyncio
324 | async def test_consolidation_report_completeness(self, consolidator):
325 | """Test that consolidation reports contain all expected information."""
326 | report = await consolidator.consolidate("weekly")
327 |
328 | # Check all required fields
329 | required_fields = [
330 | "time_horizon", "start_time", "end_time", "memories_processed",
331 | "associations_discovered", "clusters_created", "memories_compressed",
332 | "memories_archived", "errors", "performance_metrics"
333 | ]
334 |
335 | for field in required_fields:
336 | assert hasattr(report, field), f"Missing field: {field}"
337 | assert getattr(report, field) is not None, f"Field {field} is None"
338 |
339 | # Check performance metrics
340 | perf_metrics = report.performance_metrics
341 | assert "duration_seconds" in perf_metrics
342 | assert "memories_per_second" in perf_metrics
343 | assert "success" in perf_metrics
344 |
345 | @pytest.mark.asyncio
346 | async def test_storage_backend_integration(self, consolidator, mock_storage):
347 | """Test integration with storage backend methods."""
348 | # Run consolidation
349 | report = await consolidator.consolidate("monthly")
350 |
351 | # Verify storage integration worked (memories were processed)
352 | assert report.memories_processed >= 0
353 | assert isinstance(report.performance_metrics, dict)
354 |
355 | # Verify storage backend has the expected methods
356 | assert hasattr(mock_storage, 'get_all_memories')
357 | assert hasattr(mock_storage, 'get_memories_by_time_range')
358 | assert hasattr(mock_storage, 'get_memory_connections')
359 | assert hasattr(mock_storage, 'get_access_patterns')
360 | assert hasattr(mock_storage, 'update_memory')
361 |
362 | @pytest.mark.asyncio
363 | async def test_configuration_impact(self, mock_storage):
364 | """Test that configuration changes affect consolidation behavior."""
365 | # Create two different configurations
366 | config1 = type('Config', (), {
367 | 'decay_enabled': True,
368 | 'associations_enabled': True,
369 | 'clustering_enabled': True,
370 | 'compression_enabled': True,
371 | 'forgetting_enabled': True,
372 | 'retention_periods': {'standard': 30},
373 | 'min_similarity': 0.3,
374 | 'max_similarity': 0.7,
375 | 'max_pairs_per_run': 50,
376 | 'min_cluster_size': 3,
377 | 'clustering_algorithm': 'simple',
378 | 'max_summary_length': 200,
379 | 'preserve_originals': True,
380 | 'relevance_threshold': 0.1,
381 | 'access_threshold_days': 30,
382 | 'archive_location': None
383 | })()
384 |
385 | config2 = type('Config', (), {
386 | 'decay_enabled': False,
387 | 'associations_enabled': False,
388 | 'clustering_enabled': False,
389 | 'compression_enabled': False,
390 | 'forgetting_enabled': False,
391 | 'retention_periods': {'standard': 30},
392 | 'min_similarity': 0.3,
393 | 'max_similarity': 0.7,
394 | 'max_pairs_per_run': 50,
395 | 'min_cluster_size': 3,
396 | 'clustering_algorithm': 'simple',
397 | 'max_summary_length': 200,
398 | 'preserve_originals': True,
399 | 'relevance_threshold': 0.1,
400 | 'access_threshold_days': 30,
401 | 'archive_location': None
402 | })()
403 |
404 | consolidator1 = DreamInspiredConsolidator(mock_storage, config1)
405 | consolidator2 = DreamInspiredConsolidator(mock_storage, config2)
406 |
407 | # Both should work, but may produce different results
408 | report1 = await consolidator1.consolidate("weekly")
409 | report2 = await consolidator2.consolidate("weekly")
410 |
411 | assert isinstance(report1, ConsolidationReport)
412 | assert isinstance(report2, ConsolidationReport)
413 |
414 | # With disabled features, the second consolidator might process differently
415 | # but both should complete successfully
416 | assert report1.performance_metrics["success"] is True
417 | assert report2.performance_metrics["success"] is True
```
--------------------------------------------------------------------------------
/docs/pr-graphql-integration.md:
--------------------------------------------------------------------------------
```markdown
1 | # PR Review Thread Management with GraphQL
2 |
3 | **Status:** ✅ Implemented in v8.20.0
4 | **Motivation:** Eliminate manual "mark as resolved" clicks, reduce PR review friction
5 | **Key Benefit:** Automatic thread resolution when code is fixed
6 |
7 | ---
8 |
9 | ## Table of Contents
10 |
11 | 1. [Overview](#overview)
12 | 2. [Why GraphQL?](#why-graphql)
13 | 3. [Components](#components)
14 | 4. [Usage Guide](#usage-guide)
15 | 5. [Integration with Automation](#integration-with-automation)
16 | 6. [Troubleshooting](#troubleshooting)
17 | 7. [API Reference](#api-reference)
18 |
19 | ---
20 |
21 | ## Overview
22 |
23 | This system provides **automated PR review thread management** using GitHub's GraphQL API. It eliminates the manual work of resolving review threads by:
24 |
25 | 1. **Detecting** which code changes address review comments
26 | 2. **Automatically resolving** threads for fixed code
27 | 3. **Adding explanatory comments** with commit references
28 | 4. **Tracking thread status** across review iterations
29 |
30 | ### Problem Solved
31 |
32 | **Before:**
33 | ```bash
34 | # Manual workflow (time-consuming, error-prone)
35 | 1. Gemini reviews code → creates 30 inline comments
36 | 2. You fix all issues → push commit
37 | 3. Manually click "Resolve" 30 times on GitHub web UI
38 | 4. Trigger new review: gh pr comment $PR --body "/gemini review"
39 | 5. Repeat...
40 | ```
41 |
42 | **After:**
43 | ```bash
44 | # Automated workflow (zero manual clicks)
45 | 1. Gemini reviews code → creates 30 inline comments
46 | 2. You fix all issues → push commit
47 | 3. Auto-resolve: bash scripts/pr/resolve_threads.sh $PR HEAD --auto
48 | 4. Trigger new review: gh pr comment $PR --body "/gemini review"
49 | 5. Repeat...
50 | ```
51 |
52 | Even better with `auto_review.sh` - it auto-resolves threads after each fix iteration!
53 |
54 | ---
55 |
56 | ## Why GraphQL?
57 |
58 | ### GitHub API Limitation
59 |
60 | **Critical discovery:** GitHub's REST API **cannot** resolve PR review threads.
61 |
62 | ```bash
63 | # ❌ REST API - No thread resolution endpoint
64 | gh api repos/OWNER/REPO/pulls/PR_NUMBER/comments
65 | # Can list comments, but cannot resolve threads
66 |
67 | # ✅ GraphQL API - Full thread management
68 | gh api graphql -f query='mutation { resolveReviewThread(...) }'
69 | # Can query threads, resolve them, add replies
70 | ```
71 |
72 | ### GraphQL Advantages
73 |
74 | | Feature | REST API | GraphQL API |
75 | |---------|----------|-------------|
76 | | **List review comments** | ✅ Yes | ✅ Yes |
77 | | **Get thread status** | ❌ No | ✅ Yes (`isResolved`, `isOutdated`) |
78 | | **Resolve threads** | ❌ No | ✅ Yes (`resolveReviewThread` mutation) |
79 | | **Add thread replies** | ❌ Limited | ✅ Yes (`addPullRequestReviewThreadReply`) |
80 | | **Thread metadata** | ❌ No | ✅ Yes (line, path, diffSide) |
81 |
82 | ---
83 |
84 | ## Components
85 |
86 | ### 1. GraphQL Helpers Library
87 |
88 | **File:** `scripts/pr/lib/graphql_helpers.sh`
89 | **Purpose:** Reusable GraphQL operations for PR review threads
90 |
91 | **Key Functions:**
92 |
93 | ```bash
94 | # Get all review threads for a PR
95 | get_review_threads <PR_NUMBER>
96 |
97 | # Resolve a thread (with optional comment)
98 | resolve_review_thread <THREAD_ID> [COMMENT]
99 |
100 | # Add a reply to a thread
101 | add_thread_reply <THREAD_ID> <COMMENT>
102 |
103 | # Check if a line was modified in a commit
104 | was_line_modified <FILE_PATH> <LINE_NUMBER> <COMMIT_SHA>
105 |
106 | # Get thread statistics
107 | get_thread_stats <PR_NUMBER>
108 | count_unresolved_threads <PR_NUMBER>
109 |
110 | # Verify gh CLI supports GraphQL
111 | check_graphql_support
112 | ```
113 |
114 | **GraphQL Queries Used:**
115 |
116 | 1. **Query review threads:**
117 | ```graphql
118 | query($pr: Int!, $owner: String!, $repo: String!) {
119 | repository(owner: $owner, name: $repo) {
120 | pullRequest(number: $pr) {
121 | reviewThreads(first: 100) {
122 | nodes {
123 | id
124 | isResolved
125 | isOutdated
126 | path
127 | line
128 | comments(first: 10) {
129 | nodes {
130 | id
131 | author { login }
132 | body
133 | createdAt
134 | }
135 | }
136 | }
137 | }
138 | }
139 | }
140 | }
141 | ```
142 |
143 | 2. **Resolve a thread:**
144 | ```graphql
145 | mutation($threadId: ID!) {
146 | resolveReviewThread(input: {threadId: $threadId}) {
147 | thread {
148 | id
149 | isResolved
150 | }
151 | }
152 | }
153 | ```
154 |
155 | 3. **Add thread reply:**
156 | ```graphql
157 | mutation($threadId: ID!, $body: String!) {
158 | addPullRequestReviewThreadReply(input: {
159 | pullRequestReviewThreadId: $threadId
160 | body: $body
161 | }) {
162 | comment { id }
163 | }
164 | }
165 | ```
166 |
167 | ### 2. Smart Thread Resolution Tool
168 |
169 | **File:** `scripts/pr/resolve_threads.sh`
170 | **Purpose:** Automatically resolve threads when code is fixed
171 |
172 | **Usage:**
173 |
174 | ```bash
175 | # Interactive mode (prompts for each thread)
176 | bash scripts/pr/resolve_threads.sh <PR_NUMBER> [COMMIT_SHA]
177 |
178 | # Automatic mode (no prompts)
179 | bash scripts/pr/resolve_threads.sh <PR_NUMBER> HEAD --auto
180 |
181 | # Example
182 | bash scripts/pr/resolve_threads.sh 212 HEAD --auto
183 | ```
184 |
185 | **Decision Logic:**
186 |
187 | ```bash
188 | For each unresolved thread:
189 | 1. Is the file modified in this commit?
190 | → Yes: Check if the specific line was changed
191 | → Yes: Resolve with "Line X modified in commit ABC"
192 | → No: Skip
193 | → No: Check if thread is marked "outdated" by GitHub
194 | → Yes: Resolve with "Thread outdated by subsequent commits"
195 | → No: Skip
196 | ```
197 |
198 | **Resolution Comment Format:**
199 |
200 | ```markdown
201 | ✅ Resolved: Line 123 in file.py was modified in commit abc1234
202 |
203 | Verified by automated thread resolution script.
204 | ```
205 |
206 | ### 3. Thread Status Display
207 |
208 | **File:** `scripts/pr/thread_status.sh`
209 | **Purpose:** Display comprehensive thread status with filtering
210 |
211 | **Usage:**
212 |
213 | ```bash
214 | # Show all threads with summary
215 | bash scripts/pr/thread_status.sh <PR_NUMBER>
216 |
217 | # Show only unresolved threads
218 | bash scripts/pr/thread_status.sh <PR_NUMBER> --unresolved
219 |
220 | # Show only resolved threads
221 | bash scripts/pr/thread_status.sh <PR_NUMBER> --resolved
222 |
223 | # Show only outdated threads
224 | bash scripts/pr/thread_status.sh <PR_NUMBER> --outdated
225 |
226 | # Example
227 | bash scripts/pr/thread_status.sh 212 --unresolved
228 | ```
229 |
230 | **Output Format:**
231 |
232 | ```
233 | ========================================
234 | PR Review Thread Status
235 | ========================================
236 | PR Number: #212
237 | Filter: unresolved
238 |
239 | ========================================
240 | Summary
241 | ========================================
242 | Total Threads: 45
243 | Resolved: 39
244 | Unresolved: 6
245 | Outdated: 12
246 |
247 | ========================================
248 | Thread Details
249 | ========================================
250 |
251 | ○ Thread #1
252 | Status: UNRESOLVED | OUTDATED
253 | File: scripts/pr/auto_review.sh:89
254 | Side: RIGHT
255 | Author: gemini-code-assist[bot]
256 | Created: 2025-11-08T12:30:45Z
257 | Comments: 1
258 | "Variable $review_comments is undefined. Define it before use..."
259 | Thread ID: MDEyOlB1bGxSZXF1ZXN...
260 |
261 | ...
262 | ```
263 |
264 | ### 4. Integration with Auto-Review
265 |
266 | **File:** `scripts/pr/auto_review.sh` (enhanced)
267 | **Added functionality:**
268 |
269 | 1. **Startup:** Check GraphQL availability
270 | ```bash
271 | GraphQL Thread Resolution: Enabled
272 | ```
273 |
274 | 2. **Per-iteration:** Display thread stats
275 | ```bash
276 | Review Threads: 45 total, 30 resolved, 15 unresolved
277 | ```
278 |
279 | 3. **After pushing fixes:** Auto-resolve threads
280 | ```bash
281 | Resolving review threads for fixed code...
282 | ✅ Review threads auto-resolved (8 threads)
283 | ```
284 |
285 | ### 5. Integration with Watch Mode
286 |
287 | **File:** `scripts/pr/watch_reviews.sh` (enhanced)
288 | **Added functionality:**
289 |
290 | 1. **Startup:** Check GraphQL availability
291 | ```bash
292 | GraphQL Thread Tracking: Enabled
293 | ```
294 |
295 | 2. **Per-check:** Display thread stats
296 | ```bash
297 | Review Threads: 45 total, 30 resolved, 15 unresolved
298 | ```
299 |
300 | 3. **On new review:** Show unresolved thread details
301 | ```bash
302 | Thread Status:
303 | [Displays thread_status.sh --unresolved output]
304 |
305 | Options:
306 | 1. View detailed thread status:
307 | bash scripts/pr/thread_status.sh 212
308 | ...
309 | ```
310 |
311 | ---
312 |
313 | ## Usage Guide
314 |
315 | ### Basic Workflow
316 |
317 | **1. Check thread status:**
318 |
319 | ```bash
320 | bash scripts/pr/thread_status.sh 212
321 | ```
322 |
323 | **2. Fix issues and push:**
324 |
325 | ```bash
326 | # Fix code based on review comments
327 | git add .
328 | git commit -m "fix: address review feedback"
329 | git push
330 | ```
331 |
332 | **3. Resolve threads for fixed code:**
333 |
334 | ```bash
335 | # Automatic resolution
336 | bash scripts/pr/resolve_threads.sh 212 HEAD --auto
337 |
338 | # Interactive resolution (with prompts)
339 | bash scripts/pr/resolve_threads.sh 212 HEAD
340 | ```
341 |
342 | **4. Trigger new review:**
343 |
344 | ```bash
345 | gh pr comment 212 --body "/gemini review"
346 | ```
347 |
348 | ### Integrated Workflow (Recommended)
349 |
350 | **Use auto_review.sh - it handles everything:**
351 |
352 | ```bash
353 | bash scripts/pr/auto_review.sh 212 5 true
354 | ```
355 |
356 | This will:
357 | - Fetch review feedback
358 | - Categorize issues
359 | - Generate fixes
360 | - Apply and push fixes
361 | - **Auto-resolve threads** ← New!
362 | - Wait for next review
363 | - Repeat
364 |
365 | **Use watch_reviews.sh for monitoring:**
366 |
367 | ```bash
368 | bash scripts/pr/watch_reviews.sh 212 120
369 | ```
370 |
371 | This will:
372 | - Check for new reviews every 120s
373 | - **Display thread status** ← New!
374 | - Show unresolved threads when reviews arrive
375 | - Optionally trigger auto_review.sh
376 |
377 | ### Advanced Usage
378 |
379 | **Manual thread resolution with custom comment:**
380 |
381 | ```bash
382 | # Interactive mode allows custom comments
383 | bash scripts/pr/resolve_threads.sh 212 HEAD
384 |
385 | # When prompted:
386 | Resolve this thread? (y/N): y
387 | Add custom comment? (leave empty for auto): Fixed by refactoring storage backend
388 |
389 | # Result:
390 | ✅ Fixed by refactoring storage backend
391 | ```
392 |
393 | **Query thread info programmatically:**
394 |
395 | ```bash
396 | # Source the helpers
397 | source scripts/pr/lib/graphql_helpers.sh
398 |
399 | # Get all threads as JSON
400 | threads=$(get_review_threads 212)
401 |
402 | # Extract specific data
403 | echo "$threads" | jq '.data.repository.pullRequest.reviewThreads.nodes[] |
404 | select(.isResolved == false) |
405 | {file: .path, line: .line, comment: .comments.nodes[0].body}'
406 | ```
407 |
408 | **Check specific file's threads:**
409 |
410 | ```bash
411 | source scripts/pr/lib/graphql_helpers.sh
412 |
413 | # Get threads for specific file
414 | get_unresolved_threads_for_file 212 "scripts/pr/auto_review.sh"
415 | ```
416 |
417 | ---
418 |
419 | ## Integration with Automation
420 |
421 | ### Gemini PR Automator Agent
422 |
423 | The gemini-pr-automator agent (`.claude/agents/gemini-pr-automator.md`) now includes GraphQL thread management:
424 |
425 | **Phase 1: Initial PR Creation**
426 | ```bash
427 | # After creating PR, start watch mode with GraphQL tracking
428 | bash scripts/pr/watch_reviews.sh $PR_NUMBER 180 &
429 | ```
430 |
431 | **Phase 2: Review Iteration**
432 | ```bash
433 | # Auto-review now auto-resolves threads
434 | bash scripts/pr/auto_review.sh $PR_NUMBER 5 true
435 | # Includes:
436 | # - Fix issues
437 | # - Push commits
438 | # - Resolve threads ← Automatic!
439 | # - Trigger new review
440 | ```
441 |
442 | **Phase 3: Manual Fixes**
443 | ```bash
444 | # After manual fixes
445 | git push
446 | bash scripts/pr/resolve_threads.sh $PR_NUMBER HEAD --auto
447 | gh pr comment $PR_NUMBER --body "/gemini review"
448 | ```
449 |
450 | ### Pre-commit Integration (Future)
451 |
452 | **Potential enhancement:** Warn about unresolved threads before allowing new commits
453 |
454 | ```bash
455 | # In .git/hooks/pre-commit
456 | if [ -n "$PR_BRANCH" ]; then
457 | unresolved=$(bash scripts/pr/thread_status.sh $PR_NUMBER --unresolved | grep "Unresolved:" | awk '{print $2}')
458 |
459 | if [ "$unresolved" -gt 0 ]; then
460 | echo "⚠️ Warning: $unresolved unresolved review threads"
461 | echo "Consider resolving before committing new changes"
462 | fi
463 | fi
464 | ```
465 |
466 | ---
467 |
468 | ## Troubleshooting
469 |
470 | ### Issue 1: GraphQL helpers not found
471 |
472 | **Symptom:**
473 | ```
474 | Warning: GraphQL helpers not available, thread auto-resolution disabled
475 | ```
476 |
477 | **Cause:** `scripts/pr/lib/graphql_helpers.sh` not found
478 |
479 | **Fix:**
480 | ```bash
481 | # Verify file exists
482 | ls -la scripts/pr/lib/graphql_helpers.sh
483 |
484 | # If missing, re-pull from main branch
485 | git checkout main -- scripts/pr/lib/
486 | ```
487 |
488 | ### Issue 2: gh CLI doesn't support GraphQL
489 |
490 | **Symptom:**
491 | ```
492 | Error: GitHub CLI version X.Y.Z is too old
493 | GraphQL support requires v2.20.0 or later
494 | ```
495 |
496 | **Fix:**
497 | ```bash
498 | # Update gh CLI
499 | gh upgrade
500 |
501 | # Or install latest from https://cli.github.com/
502 | ```
503 |
504 | ### Issue 3: Thread resolution fails
505 |
506 | **Symptom:**
507 | ```
508 | ❌ Failed to resolve
509 | ```
510 |
511 | **Causes and fixes:**
512 |
513 | 1. **Invalid thread ID:**
514 | ```bash
515 | # Verify thread exists
516 | bash scripts/pr/thread_status.sh $PR_NUMBER
517 | ```
518 |
519 | 2. **Network issues:**
520 | ```bash
521 | # Check GitHub connectivity
522 | gh auth status
523 | gh api graphql -f query='query { viewer { login } }'
524 | ```
525 |
526 | 3. **Permissions:**
527 | ```bash
528 | # Ensure you have write access to the repository
529 | gh repo view --json viewerPermission
530 | ```
531 |
532 | ### Issue 4: Threads not auto-resolving during auto_review
533 |
534 | **Symptom:**
535 | Auto-review runs but threads remain unresolved
536 |
537 | **Debug steps:**
538 |
539 | 1. **Check GraphQL availability:**
540 | ```bash
541 | bash scripts/pr/auto_review.sh 212 1 true 2>&1 | grep "GraphQL"
542 | # Should show: GraphQL Thread Resolution: Enabled
543 | ```
544 |
545 | 2. **Verify thread resolution script works:**
546 | ```bash
547 | bash scripts/pr/resolve_threads.sh 212 HEAD --auto
548 | # Should resolve threads if any changes match
549 | ```
550 |
551 | 3. **Check commit SHA detection:**
552 | ```bash
553 | git rev-parse HEAD
554 | # Should return valid SHA
555 | ```
556 |
557 | ### Issue 5: "No threads needed resolution" when threads exist
558 |
559 | **Symptom:**
560 | ```
561 | ℹ️ No threads needed resolution
562 | ```
563 |
564 | **Cause:** Threads reference lines that weren't modified in the commit
565 |
566 | **Explanation:**
567 |
568 | The tool only resolves threads for **code that was actually changed**:
569 |
570 | ```bash
571 | # Thread on line 89 of file.py
572 | # Your commit modified lines 100-120
573 | # → Thread NOT resolved (line 89 unchanged)
574 |
575 | # Thread on line 105 of file.py
576 | # Your commit modified lines 100-120
577 | # → Thread RESOLVED (line 105 changed)
578 | ```
579 |
580 | **Fix:** Either:
581 | 1. Modify the code that the thread references
582 | 2. Manually resolve via GitHub web UI if thread is no longer relevant
583 | 3. Wait for thread to become "outdated" (GitHub marks it automatically after subsequent commits)
584 |
585 | ---
586 |
587 | ## API Reference
588 |
589 | ### GraphQL Helper Functions
590 |
591 | #### `get_review_threads <PR_NUMBER>`
592 |
593 | **Description:** Fetch all review threads for a PR
594 |
595 | **Returns:** JSON with thread data
596 |
597 | **Example:**
598 | ```bash
599 | source scripts/pr/lib/graphql_helpers.sh
600 | threads=$(get_review_threads 212)
601 | echo "$threads" | jq '.data.repository.pullRequest.reviewThreads.nodes | length'
602 | # Output: 45
603 | ```
604 |
605 | #### `resolve_review_thread <THREAD_ID> [COMMENT]`
606 |
607 | **Description:** Resolve a review thread with optional comment
608 |
609 | **Parameters:**
610 | - `THREAD_ID`: GraphQL node ID (e.g., `MDEyOlB1bGxSZXF1ZXN...`)
611 | - `COMMENT`: Optional explanatory comment
612 |
613 | **Returns:** 0 on success, 1 on failure
614 |
615 | **Example:**
616 | ```bash
617 | resolve_review_thread "MDEyOlB1bGxSZXF1ZXN..." "Fixed in commit abc1234"
618 | ```
619 |
620 | #### `add_thread_reply <THREAD_ID> <COMMENT>`
621 |
622 | **Description:** Add a reply to a thread without resolving
623 |
624 | **Parameters:**
625 | - `THREAD_ID`: GraphQL node ID
626 | - `COMMENT`: Reply text (required)
627 |
628 | **Returns:** 0 on success, 1 on failure
629 |
630 | **Example:**
631 | ```bash
632 | add_thread_reply "MDEyOlB1bGxSZXF1ZXN..." "Working on this now, will fix in next commit"
633 | ```
634 |
635 | #### `was_line_modified <FILE_PATH> <LINE_NUMBER> <COMMIT_SHA>`
636 |
637 | **Description:** Check if a specific line was modified in a commit
638 |
639 | **Parameters:**
640 | - `FILE_PATH`: Relative path to file
641 | - `LINE_NUMBER`: Line number to check
642 | - `COMMIT_SHA`: Commit to check (e.g., `HEAD`, `abc1234`)
643 |
644 | **Returns:** 0 if modified, 1 if not
645 |
646 | **Example:**
647 | ```bash
648 | if was_line_modified "scripts/pr/auto_review.sh" 89 "HEAD"; then
649 | echo "Line 89 was modified"
650 | fi
651 | ```
652 |
653 | #### `get_thread_stats <PR_NUMBER>`
654 |
655 | **Description:** Get summary statistics for PR review threads
656 |
657 | **Returns:** JSON with counts
658 |
659 | **Example:**
660 | ```bash
661 | stats=$(get_thread_stats 212)
662 | echo "$stats" | jq '.unresolved'
663 | # Output: 6
664 | ```
665 |
666 | #### `count_unresolved_threads <PR_NUMBER>`
667 |
668 | **Description:** Get count of unresolved threads
669 |
670 | **Returns:** Integer count
671 |
672 | **Example:**
673 | ```bash
674 | count=$(count_unresolved_threads 212)
675 | echo "Unresolved threads: $count"
676 | # Output: Unresolved threads: 6
677 | ```
678 |
679 | ---
680 |
681 | ## Best Practices
682 |
683 | ### 1. Use Auto-Resolution Conservatively
684 |
685 | **Do auto-resolve when:**
686 | - ✅ You fixed the exact code mentioned in the comment
687 | - ✅ The commit directly addresses the review feedback
688 | - ✅ Tests pass after the fix
689 |
690 | **Don't auto-resolve when:**
691 | - ❌ Unsure if the fix fully addresses the concern
692 | - ❌ The review comment asks a question (not a fix request)
693 | - ❌ Breaking changes involved (needs discussion)
694 |
695 | ### 2. Add Meaningful Comments
696 |
697 | **Good resolution comments:**
698 | ```
699 | ✅ Fixed: Refactored using async/await pattern as suggested
700 | ✅ Resolved: Added type hints for all parameters
701 | ✅ Addressed: Extracted helper function to reduce complexity
702 | ```
703 |
704 | **Bad resolution comments:**
705 | ```
706 | ❌ Done
707 | ❌ Fixed
708 | ❌ OK
709 | ```
710 |
711 | ### 3. Verify Before Auto-Resolving
712 |
713 | ```bash
714 | # 1. Check what will be resolved
715 | bash scripts/pr/resolve_threads.sh 212 HEAD
716 |
717 | # Review the prompts, then run in auto mode
718 | bash scripts/pr/resolve_threads.sh 212 HEAD --auto
719 | ```
720 |
721 | ### 4. Monitor Thread Status
722 |
723 | ```bash
724 | # Regular check during review cycle
725 | bash scripts/pr/thread_status.sh 212 --unresolved
726 |
727 | # Track progress
728 | bash scripts/pr/thread_status.sh 212
729 | # Shows: 45 total, 39 resolved, 6 unresolved
730 | ```
731 |
732 | ---
733 |
734 | ## Performance Considerations
735 |
736 | ### API Rate Limits
737 |
738 | GitHub GraphQL API has rate limits:
739 | - **Authenticated:** 5,000 points per hour
740 | - **Points per query:** ~1 point for simple queries, ~10 for complex
741 |
742 | **Our usage:**
743 | - `get_review_threads`: ~5 points (fetches 100 threads with comments)
744 | - `resolve_review_thread`: ~1 point
745 | - `get_thread_stats`: ~5 points
746 |
747 | **Typical PR with 30 threads:**
748 | - Initial status check: 5 points
749 | - Resolve 30 threads: 30 points
750 | - Final status check: 5 points
751 | - **Total: ~40 points** (0.8% of hourly limit)
752 |
753 | **Conclusion:** Rate limits not a concern for typical PR workflows.
754 |
755 | ### Network Latency
756 |
757 | - GraphQL API calls: ~200-500ms each
758 | - Auto-resolving 30 threads: ~1-2 seconds total
759 | - Minimal impact on review cycle time
760 |
761 | ---
762 |
763 | ## Future Enhancements
764 |
765 | ### 1. Bulk Thread Operations
766 |
767 | **Idea:** Resolve all threads for a file in one mutation
768 |
769 | ```bash
770 | # Current: 30 API calls for 30 threads
771 | for thread in $threads; do
772 | resolve_review_thread "$thread"
773 | done
774 |
775 | # Future: 1 API call for 30 threads
776 | resolve_threads_batch "${thread_ids[@]}"
777 | ```
778 |
779 | ### 2. Smart Thread Filtering
780 |
781 | **Idea:** Only show threads relevant to recent commits
782 |
783 | ```bash
784 | bash scripts/pr/thread_status.sh 212 --since="2 hours ago"
785 | bash scripts/pr/thread_status.sh 212 --author="gemini-code-assist[bot]"
786 | ```
787 |
788 | ### 3. Thread Diff View
789 |
790 | **Idea:** Show what changed for each thread
791 |
792 | ```bash
793 | bash scripts/pr/thread_diff.sh 212
794 | # Shows:
795 | # Thread #1: scripts/pr/auto_review.sh:89
796 | # Before: review_comments=$(undefined)
797 | # After: review_comments=$(gh api "repos/$REPO/pulls/$PR_NUMBER/comments" | ...)
798 | # Status: Fixed ✅
799 | ```
800 |
801 | ### 4. Pre-Push Hook Integration
802 |
803 | **Idea:** Warn before pushing if unresolved threads exist
804 |
805 | ```bash
806 | # .git/hooks/pre-push
807 | unresolved=$(count_unresolved_threads $PR_NUMBER)
808 | if [ "$unresolved" -gt 0 ]; then
809 | echo "⚠️ $unresolved unresolved threads"
810 | read -p "Continue? (y/N): " response
811 | fi
812 | ```
813 |
814 | ---
815 |
816 | ## Related Documentation
817 |
818 | - **Gemini PR Automator:** `.claude/agents/gemini-pr-automator.md`
819 | - **Code Quality Guard:** `.claude/agents/code-quality-guard.md`
820 | - **Auto-Review Script:** `scripts/pr/auto_review.sh`
821 | - **Watch Mode Script:** `scripts/pr/watch_reviews.sh`
822 | - **GitHub GraphQL API:** https://docs.github.com/en/graphql
823 |
824 | ---
825 |
826 | **Last Updated:** 2025-11-08
827 | **Version:** 8.20.0
828 | **Maintainer:** MCP Memory Service Team
829 |
```
--------------------------------------------------------------------------------
/.claude/agents/github-release-manager.md:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | name: github-release-manager
3 | description: Use this agent when you need to manage the complete GitHub release workflow, including version management, documentation updates, branch management, PR creation, issue tracking, and post-release communication. This agent should be invoked proactively in the following scenarios:\n\n<example>\nContext: User has just completed a significant feature implementation or bug fix.\nuser: "I've finished implementing the new caching system for the memory service"\nassistant: "Great work! Let me use the github-release-manager agent to handle the complete release workflow for this change."\n<commentary>\nThe agent will analyze the changes, determine appropriate version bump, update CHANGELOG.md and README.md, create a feature branch if needed, push changes, and create a PR for Gemini Code Assist review.\n</commentary>\n</example>\n\n<example>\nContext: Multiple commits have been merged to the main branch since the last release.\nuser: "What's the status of our pending changes?"\nassistant: "Let me check the release status using the github-release-manager agent."\n<commentary>\nThe agent will review commits since last release, assess if a new release is warranted, determine version bump type, and initiate the release workflow if appropriate.\n</commentary>\n</example>\n\n<example>\nContext: An issue has been marked as fixed in a recent commit.\nuser: "I've pushed the fix for issue #123"\nassistant: "Excellent! I'll use the github-release-manager agent to track this fix through the release process and handle issue closure."\n<commentary>\nThe agent will monitor the fix through PR review, release tagging, and automatically close the issue with a grateful comment once released.\n</commentary>\n</example>\n\n<example>\nContext: User mentions planning or refactoring work.\nuser: "We should refactor the storage backend architecture"\nassistant: "I'll use the github-release-manager agent to analyze open issues and create a refactoring plan."\n<commentary>\nThe agent will review related issues, assess technical debt, and create a structured plan for the refactoring work including branch strategy and milestone planning.\n</commentary>\n</example>\n\n<example>\nContext: Proactive monitoring after a work session.\nuser: "I think we're done for today"\nassistant: "Before we wrap up, let me use the github-release-manager agent to check if any release actions are needed based on today's work."\n<commentary>\nThe agent proactively reviews the session's commits, determines if version bumps or documentation updates are needed, and can initiate the release workflow automatically.\n</commentary>\n</example>
4 | model: sonnet
5 | color: purple
6 | ---
7 |
8 | You are an elite GitHub Release Manager, a specialized AI agent with deep expertise in semantic versioning, release engineering, documentation management, and issue lifecycle management. Your mission is to orchestrate the complete publishing workflow for the MCP Memory Service project with precision, consistency, and professionalism.
9 |
10 | ## Core Responsibilities
11 |
12 | You are responsible for the entire release lifecycle:
13 |
14 | 1. **Version Management**: Analyze commits and changes to determine appropriate semantic version bumps (major.minor.patch) following semver principles strictly
15 | 2. **Documentation Curation**: Update CHANGELOG.md with detailed, well-formatted entries and update README.md when features affect user-facing functionality
16 | 3. **Branch Strategy**: Decide when to create feature/fix branches vs. working directly on main/develop, following the project's git workflow
17 | 4. **Release Orchestration**: Create git tags, GitHub releases with comprehensive release notes, and ensure all artifacts are properly published
18 | 5. **PR Management**: Create pull requests with detailed descriptions and coordinate with Gemini Code Assist for automated reviews
19 | 6. **Issue Lifecycle**: Monitor issues, plan refactoring work, provide grateful closure comments with context, and maintain issue hygiene
20 |
21 | ## Decision-Making Framework
22 |
23 | ### Version Bump Determination
24 |
25 | Analyze changes using these criteria:
26 |
27 | - **MAJOR (x.0.0)**: Breaking API changes, removed features, incompatible architecture changes
28 | - **MINOR (0.x.0)**: New features, significant enhancements, new capabilities (backward compatible)
29 | - **PATCH (0.0.x)**: Bug fixes, performance improvements, documentation updates, minor tweaks
30 |
31 | Consider the project context from CLAUDE.md:
32 | - Storage backend changes may warrant MINOR bumps
33 | - MCP protocol changes may warrant MAJOR bumps
34 | - Hook system changes should be evaluated for breaking changes
35 | - Performance improvements >20% may warrant MINOR bumps
36 |
37 | ### Branch Strategy Decision Matrix
38 |
39 | **Create a new branch when:**
40 | - Feature development will take multiple commits
41 | - Changes are experimental or require review before merge
42 | - Working on a fix for a specific issue that needs isolated testing
43 | - Multiple developers might work on related changes
44 | - Changes affect critical systems (storage backends, MCP protocol)
45 |
46 | **Work directly on main/develop when:**
47 | - Hot fixes for critical bugs
48 | - Documentation-only updates
49 | - Version bump commits
50 | - Single-commit changes that are well-tested
51 |
52 | ### Documentation Update Strategy
53 |
54 | Follow the project's Documentation Decision Matrix from CLAUDE.md:
55 |
56 | **CHANGELOG.md** (Always update for):
57 | - Bug fixes with issue references
58 | - New features with usage examples
59 | - Performance improvements with metrics
60 | - Configuration changes with migration notes
61 | - Breaking changes with upgrade guides
62 |
63 | **README.md** (Update when):
64 | - New features affect installation or setup
65 | - Command-line interface changes
66 | - New environment variables or configuration options
67 | - Architecture changes affect user understanding
68 |
69 | **CLAUDE.md** (Update when):
70 | - New commands or workflows are introduced
71 | - Development guidelines change
72 | - Troubleshooting procedures are discovered
73 |
74 | ### PR Creation and Review Workflow
75 |
76 | When creating pull requests:
77 |
78 | 1. **Title Format**: Use conventional commits format (feat:, fix:, docs:, refactor:, perf:, test:)
79 | 2. **Description Template**:
80 | ```markdown
81 | ## Changes
82 | - Detailed list of changes
83 |
84 | ## Motivation
85 | - Why these changes are needed
86 |
87 | ## Testing
88 | - How changes were tested
89 |
90 | ## Related Issues
91 | - Fixes #123, Closes #456
92 |
93 | ## Checklist
94 | - [ ] Version bumped in __init__.py and pyproject.toml
95 | - [ ] CHANGELOG.md updated
96 | - [ ] README.md updated (if needed)
97 | - [ ] Tests added/updated
98 | - [ ] Documentation updated
99 | ```
100 |
101 | 3. **Gemini Review Coordination**: After PR creation, wait for Gemini Code Assist review, address feedback iteratively (Fix → Comment → /gemini review → Wait 1min → Repeat)
102 |
103 | ### Issue Management Protocol
104 |
105 | **Issue Tracking**:
106 | - Monitor commits for patterns: "fixes #", "closes #", "resolves #"
107 | - Auto-categorize issues: bug, feature, docs, performance, refactoring
108 | - Track issue-PR relationships for post-release closure
109 |
110 | **Refactoring Planning**:
111 | - Review open issues tagged with "refactoring" or "technical-debt"
112 | - Assess impact and priority based on:
113 | - Code complexity metrics
114 | - Frequency of related bugs
115 | - Developer pain points mentioned in issues
116 | - Performance implications
117 | - Create structured refactoring plans with milestones
118 |
119 | **Issue Closure**:
120 | - Wait until fix is released (not just merged)
121 | - Generate grateful, context-rich closure comments:
122 | ```markdown
123 | 🎉 This issue has been resolved in v{version}!
124 |
125 | **Fix Details:**
126 | - PR: #{pr_number}
127 | - Commit: {commit_hash}
128 | - CHANGELOG: [View entry](link)
129 |
130 | **What Changed:**
131 | {brief description of the fix}
132 |
133 | Thank you for reporting this issue and helping improve the MCP Memory Service!
134 | ```
135 |
136 | ## Operational Workflow
137 |
138 | ### Complete Release Procedure
139 |
140 | 1. **Pre-Release Analysis**:
141 | - Review commits since last release
142 | - Identify breaking changes, new features, bug fixes
143 | - Determine appropriate version bump
144 | - Check for open issues that will be resolved
145 |
146 | 2. **Four-File Version Bump Procedure**:
147 | - Update `src/mcp_memory_service/__init__.py` (line 50: `__version__ = "X.Y.Z"`)
148 | - Update `pyproject.toml` (line 7: `version = "X.Y.Z"`)
149 | - Update `README.md` "Latest Release" section (documented in step 3b below)
150 | - Run `uv lock` to update dependency lock file
151 | - Commit ALL FOUR files together: `git commit -m "chore: release vX.Y.Z"`
152 |
153 | **CRITICAL**: All four files must be updated in single commit for version consistency
154 |
155 | 3. **Documentation Updates** (CRITICAL - Must be done in correct order):
156 |
157 | a. **CHANGELOG.md Validation** (FIRST - Before any edits):
158 | - Run: `grep -n "^## \[" CHANGELOG.md | head -10`
159 | - Verify no duplicate version sections
160 | - Confirm newest version will be at top (after [Unreleased])
161 | - If PR merged with incorrect CHANGELOG:
162 | - FIX IMMEDIATELY before proceeding
163 | - Create separate commit: "docs: fix CHANGELOG structure"
164 | - DO NOT include fixes in release commit
165 | - See "CHANGELOG Validation Protocol" section for full validation commands
166 |
167 | b. **CHANGELOG.md Content**:
168 | - **FIRST**: Check for `## [Unreleased]` section
169 | - If found, move ALL unreleased entries into the new version section
170 | - Add new version entry following project format: `## [x.y.z] - YYYY-MM-DD`
171 | - Ensure empty `## [Unreleased]` section remains at top
172 | - Verify all changes from commits are documented
173 | - **VERIFY**: New version positioned immediately after [Unreleased]
174 | - **VERIFY**: No duplicate content from previous versions
175 |
176 | c. **README.md**:
177 | - **ALWAYS update** the "Latest Release" section near top of file
178 | - Update version number: `### 🆕 Latest Release: **vX.Y.Z** (Mon DD, YYYY)`
179 | - Update "What's New" bullet points with CHANGELOG highlights
180 | - Keep list concise (4-6 key items with emojis)
181 | - Match tone and format of existing entries
182 | - **CRITICAL**: Add the PREVIOUS version to "Previous Releases" section
183 | - Extract one-line summary from the old "Latest Release" content
184 | - Insert at TOP of Previous Releases list (reverse chronological order)
185 | - Format: `- **vX.Y.Z** - Brief description (key metric/feature)`
186 | - Maintain 5-6 most recent releases, remove oldest if list gets long
187 | - Example: `- **v8.24.1** - Test Infrastructure Improvements (27 test failures resolved, 63% → 71% pass rate)`
188 |
189 | d. **CLAUDE.md**:
190 | - **ALWAYS update** version reference in Overview section (line ~13): `> **vX.Y.Z**: Brief description...`
191 | - Add version callout in Overview section if significant changes
192 | - Update "Essential Commands" if new scripts/commands added
193 | - Update "Database Maintenance" section for new maintenance utilities
194 | - Update any workflow documentation affected by changes
195 |
196 | e. **Commit**:
197 | - Commit message: "docs: update CHANGELOG, README, and CLAUDE.md for v{version}"
198 |
199 | 4. **Branch and PR Management**:
200 | - Create feature branch if needed: `git checkout -b release/v{version}`
201 | - Push changes: `git push origin release/v{version}`
202 | - Create PR with comprehensive description
203 | - Tag PR for Gemini Code Assist review
204 | - Monitor review feedback and iterate
205 |
206 | 5. **Release Creation** (CRITICAL - Follow this exact sequence):
207 | - **Step 1**: Merge PR to develop branch
208 | - **Step 2**: Merge develop into main branch
209 | - **Step 3**: Switch to main branch: `git checkout main`
210 | - **Step 4**: Pull latest: `git pull origin main`
211 | - **Step 5**: NOW create annotated git tag on main: `git tag -a v{version} -m "Release v{version}"`
212 | - **Step 6**: Push tag: `git push origin v{version}`
213 | - **Step 7**: Create GitHub release with:
214 | - Tag: v{version}
215 | - Title: "v{version} - {brief description}"
216 | - Body: CHANGELOG entry + highlights
217 |
218 | **WARNING**: Do NOT create the tag before merging to main. Tags must point to main branch commits, not develop branch commits. Creating the tag on develop and then merging causes tag conflicts and incorrect release points.
219 |
220 | 6. **Post-Merge Validation** (CRITICAL - Before creating tag):
221 | - **Validate CHANGELOG Structure**:
222 | - Run: `grep -n "^## \[" CHANGELOG.md | head -10`
223 | - Verify each version appears exactly once
224 | - Confirm newest version at top (after [Unreleased])
225 | - Check no duplicate content between versions
226 | - **If CHANGELOG Issues Found**:
227 | - Create hotfix commit: `git commit -m "docs: fix CHANGELOG structure"`
228 | - Push fix: `git push origin main`
229 | - DO NOT proceed with tag creation until CHANGELOG is correct
230 | - **Verify Version Consistency**:
231 | - Check all four files have matching version (init.py, pyproject.toml, README.md, uv.lock)
232 | - Confirm git history shows clean merge to main
233 | - **Only After Validation**: Proceed to create tag in step 5 above
234 |
235 | 7. **Post-Release Actions**:
236 | - Verify GitHub Actions workflows (Docker Publish, Publish and Test, HTTP-MCP Bridge)
237 | - Retrieve related issues using memory service
238 | - Close resolved issues with grateful comments
239 | - Update project board/milestones
240 | - **Update Wiki Roadmap** (if release includes major milestones):
241 | - **When to update**: Major versions (x.0.0), significant features, architecture changes, performance breakthroughs
242 | - **How to update**: Edit [13-Development-Roadmap](https://github.com/doobidoo/mcp-memory-service/wiki/13-Development-Roadmap) directly (no PR needed)
243 | - **What to update**:
244 | - Move completed items from "Current Focus" to "Completed Milestones"
245 | - Update "Project Status" with new version number
246 | - Add notable achievements to "Recent Achievements" section
247 | - Adjust timelines if delays or accelerations occurred
248 | - **Examples of roadmap-worthy changes**:
249 | - Major version bumps (v8.x → v9.0)
250 | - New storage backends or significant backend improvements
251 | - Memory consolidation system milestones
252 | - Performance improvements >20% (page load, search, sync)
253 | - New user-facing features (dashboard, document ingestion, etc.)
254 | - **Note**: Routine patches/hotfixes don't require roadmap updates
255 |
256 | ## CHANGELOG Validation Protocol (CRITICAL)
257 |
258 | Before ANY release or documentation commit, ALWAYS validate CHANGELOG.md structure:
259 |
260 | **Validation Commands**:
261 | ```bash
262 | # 1. Check for duplicate version headers
263 | grep -n "^## \[8\." CHANGELOG.md | sort
264 | # Should show each version EXACTLY ONCE
265 |
266 | # 2. Verify chronological order (newest first)
267 | grep "^## \[" CHANGELOG.md | head -10
268 | # First should be [Unreleased], second should be highest version number
269 |
270 | # 3. Detect content duplication across versions
271 | grep -c "Hybrid Storage Sync" CHANGELOG.md
272 | # Count should match number of versions that include this feature
273 | ```
274 |
275 | **Validation Rules**:
276 | - [ ] Each version appears EXACTLY ONCE
277 | - [ ] Newest version immediately after `## [Unreleased]`
278 | - [ ] Versions in reverse chronological order (8.28.0 > 8.27.2 > 8.27.1...)
279 | - [ ] No content duplicated from other versions
280 | - [ ] New PR entries contain ONLY their own changes
281 |
282 | **Common Mistakes to Detect** (learned from PR #228 / v8.28.0):
283 | 1. **Content Duplication**: PR copies entire previous version section
284 | - Example: PR #228 copied all v8.27.0 content instead of just adding Cloudflare Tag Filtering
285 | - Detection: grep for feature names, should not appear in multiple versions
286 | 2. **Incorrect Position**: New version positioned in middle instead of top
287 | - Example: v8.28.0 appeared after v8.27.1 instead of at top
288 | - Detection: Second line after [Unreleased] must be newest version
289 | 3. **Duplicate Sections**: Same version appears multiple times
290 | - Detection: `grep "^## \[X.Y.Z\]" CHANGELOG.md` should return 1 line
291 | 4. **Date Format**: Inconsistent date format
292 | - Must be YYYY-MM-DD
293 |
294 | **If Issues Found**:
295 | 1. Remove duplicate sections completely
296 | 2. Move new version to correct position (immediately after [Unreleased])
297 | 3. Strip content that belongs to other versions
298 | 4. Verify chronological order with grep
299 | 5. Commit fix separately: `git commit -m "docs: fix CHANGELOG structure"`
300 |
301 | **Post-Merge Validation** (Before creating tag):
302 | - Run all validation commands above
303 | - If CHANGELOG issues found, create hotfix commit before tagging
304 | - DO NOT proceed with tag/release until CHANGELOG is structurally correct
305 |
306 | ## Quality Assurance
307 |
308 | **Self-Verification Checklist**:
309 | - [ ] Version follows semantic versioning strictly
310 | - [ ] All four version files updated (init, pyproject, README, lock)
311 | - [ ] **CHANGELOG.md**: `[Unreleased]` section collected and moved to version entry
312 | - [ ] **CHANGELOG.md**: Entry is detailed and well-formatted
313 | - [ ] **CHANGELOG.md**: No duplicate version sections (verified with grep)
314 | - [ ] **CHANGELOG.md**: Versions in reverse chronological order (newest first)
315 | - [ ] **CHANGELOG.md**: New version positioned immediately after [Unreleased]
316 | - [ ] **CHANGELOG.md**: No content duplicated from previous versions
317 | - [ ] **README.md**: "Latest Release" section updated with version and highlights
318 | - [ ] **README.md**: Previous version added to "Previous Releases" list (top position)
319 | - [ ] **CLAUDE.md**: New commands/utilities documented in appropriate sections
320 | - [ ] **CLAUDE.md**: Version callout added if significant changes
321 | - [ ] PR merged to develop, then develop merged to main
322 | - [ ] Git tag created on main branch (NOT develop)
323 | - [ ] Tag points to main merge commit (verify with `git log --oneline --graph --all --decorate`)
324 | - [ ] Git tag pushed to remote
325 | - [ ] GitHub release created with comprehensive notes
326 | - [ ] All related issues identified and tracked
327 | - [ ] PR description is complete and accurate
328 | - [ ] Gemini review requested and feedback addressed
329 |
330 | **Error Handling**:
331 | - If version bump is unclear, ask for clarification with specific options
332 | - If CHANGELOG conflicts exist, combine entries intelligently
333 | - If PR creation fails, provide manual instructions
334 | - If issue closure is premature, wait for release confirmation
335 |
336 | ## Communication Style
337 |
338 | - Be proactive: Suggest release actions when appropriate
339 | - Be precise: Provide exact version numbers and commit messages
340 | - Be grateful: Always thank contributors when closing issues
341 | - Be comprehensive: Include all relevant context in PRs and releases
342 | - Be cautious: Verify breaking changes before major version bumps
343 |
344 | ## Integration with Project Context
345 |
346 | You have access to project-specific context from CLAUDE.md. Always consider:
347 | - Current version from `__init__.py`
348 | - Recent changes from git history
349 | - Open issues and their priorities
350 | - Project conventions for commits and documentation
351 | - Storage backend implications of changes
352 | - MCP protocol compatibility requirements
353 |
354 | Your goal is to make the release process seamless, consistent, and professional, ensuring that every release is well-documented, properly versioned, and thoroughly communicated to users and contributors.
355 |
```
--------------------------------------------------------------------------------
/src/mcp_memory_service/mcp_server.py:
--------------------------------------------------------------------------------
```python
1 | #!/usr/bin/env python3
2 | """
3 | FastAPI MCP Server for Memory Service
4 |
5 | This module implements a native MCP server using the FastAPI MCP framework,
6 | replacing the Node.js HTTP-to-MCP bridge to resolve SSL connectivity issues
7 | and provide direct MCP protocol support.
8 |
9 | Features:
10 | - Native MCP protocol implementation using FastMCP
11 | - Direct integration with existing memory storage backends
12 | - Streamable HTTP transport for remote access
13 | - All 22 core memory operations (excluding dashboard tools)
14 | - SSL/HTTPS support with proper certificate handling
15 | """
16 |
17 | import asyncio
18 | import logging
19 | import os
20 | import socket
21 | import sys
22 | import time
23 | from collections.abc import AsyncIterator
24 | from contextlib import asynccontextmanager
25 | from dataclasses import dataclass
26 | from pathlib import Path
27 | from typing import Dict, List, Optional, Any, Union, TypedDict
28 | try:
29 | from typing import NotRequired # Python 3.11+
30 | except ImportError:
31 | from typing_extensions import NotRequired # Python 3.10
32 |
33 | # Add src to path for imports
34 | current_dir = Path(__file__).parent
35 | src_dir = current_dir.parent.parent
36 | sys.path.insert(0, str(src_dir))
37 |
38 | # FastMCP is not available in current MCP library version
39 | # This module is kept for future compatibility
40 | try:
41 | from mcp.server.fastmcp import FastMCP, Context
42 | except ImportError:
43 | logger_temp = logging.getLogger(__name__)
44 | logger_temp.warning("FastMCP not available in mcp library - mcp_server module cannot be used")
45 |
46 | # Create dummy objects for graceful degradation
47 | class _DummyFastMCP:
48 | def tool(self):
49 | """Dummy decorator that does nothing."""
50 | def decorator(func):
51 | return func
52 | return decorator
53 |
54 | FastMCP = _DummyFastMCP # type: ignore
55 | Context = None # type: ignore
56 |
57 | from mcp.types import TextContent
58 |
59 | # Import existing memory service components
60 | from .config import (
61 | STORAGE_BACKEND,
62 | CONSOLIDATION_ENABLED, EMBEDDING_MODEL_NAME, INCLUDE_HOSTNAME,
63 | SQLITE_VEC_PATH,
64 | CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_VECTORIZE_INDEX,
65 | CLOUDFLARE_D1_DATABASE_ID, CLOUDFLARE_R2_BUCKET, CLOUDFLARE_EMBEDDING_MODEL,
66 | CLOUDFLARE_LARGE_CONTENT_THRESHOLD, CLOUDFLARE_MAX_RETRIES, CLOUDFLARE_BASE_DELAY,
67 | HYBRID_SYNC_INTERVAL, HYBRID_BATCH_SIZE, HYBRID_MAX_QUEUE_SIZE,
68 | HYBRID_SYNC_ON_STARTUP, HYBRID_FALLBACK_TO_PRIMARY,
69 | CONTENT_PRESERVE_BOUNDARIES, CONTENT_SPLIT_OVERLAP, ENABLE_AUTO_SPLIT
70 | )
71 | from .storage.base import MemoryStorage
72 | from .services.memory_service import MemoryService
73 |
74 | # Configure logging
75 | logging.basicConfig(level=logging.INFO) # Default to INFO level
76 | logger = logging.getLogger(__name__)
77 |
78 | # =============================================================================
79 | # GLOBAL CACHING FOR MCP SERVER PERFORMANCE OPTIMIZATION
80 | # =============================================================================
81 | # Module-level caches to persist storage/service instances across stateless HTTP calls.
82 | # This reduces initialization overhead from ~1,810ms to <400ms on cache hits.
83 | #
84 | # Cache Keys:
85 | # - Storage: "{backend_type}:{db_path}" (e.g., "sqlite_vec:/path/to/db")
86 | # - MemoryService: storage instance ID (id(storage))
87 | #
88 | # Thread Safety:
89 | # - Uses asyncio.Lock to prevent race conditions during concurrent access
90 | #
91 | # Lifecycle:
92 | # - Cached instances persist for the lifetime of the Python process
93 | # - NOT cleared between stateless HTTP calls (intentional for performance)
94 | # - Cleaned up on process shutdown via lifespan context manager
95 |
96 | _STORAGE_CACHE: Dict[str, MemoryStorage] = {}
97 | _MEMORY_SERVICE_CACHE: Dict[int, MemoryService] = {}
98 | _CACHE_LOCK: Optional[asyncio.Lock] = None # Initialized on first use
99 | _CACHE_STATS = {
100 | "storage_hits": 0,
101 | "storage_misses": 0,
102 | "service_hits": 0,
103 | "service_misses": 0,
104 | "total_calls": 0,
105 | "initialization_times": [] # Track initialization durations for cache misses
106 | }
107 |
108 | def _get_cache_lock() -> asyncio.Lock:
109 | """Get or create the global cache lock (lazy initialization to avoid event loop issues)."""
110 | global _CACHE_LOCK
111 | if _CACHE_LOCK is None:
112 | _CACHE_LOCK = asyncio.Lock()
113 | return _CACHE_LOCK
114 |
115 | def _get_or_create_memory_service(storage: MemoryStorage) -> MemoryService:
116 | """
117 | Get cached MemoryService or create new one.
118 |
119 | Args:
120 | storage: Storage instance to use as cache key
121 |
122 | Returns:
123 | MemoryService instance (cached or newly created)
124 | """
125 | storage_id = id(storage)
126 | if storage_id in _MEMORY_SERVICE_CACHE:
127 | memory_service = _MEMORY_SERVICE_CACHE[storage_id]
128 | _CACHE_STATS["service_hits"] += 1
129 | logger.info(f"✅ MemoryService Cache HIT - Reusing service instance (storage_id: {storage_id})")
130 | else:
131 | _CACHE_STATS["service_misses"] += 1
132 | logger.info(f"❌ MemoryService Cache MISS - Creating new service instance...")
133 |
134 | # Initialize memory service with shared business logic
135 | memory_service = MemoryService(storage)
136 |
137 | # Cache the memory service instance
138 | _MEMORY_SERVICE_CACHE[storage_id] = memory_service
139 | logger.info(f"💾 Cached MemoryService instance (storage_id: {storage_id})")
140 |
141 | return memory_service
142 |
143 | def _log_cache_performance(start_time: float) -> None:
144 | """
145 | Log comprehensive cache performance statistics.
146 |
147 | Args:
148 | start_time: Timer start time to calculate total elapsed time
149 | """
150 | total_time = (time.time() - start_time) * 1000
151 | cache_hit_rate = (
152 | (_CACHE_STATS["storage_hits"] + _CACHE_STATS["service_hits"]) /
153 | (_CACHE_STATS["total_calls"] * 2) # 2 caches per call
154 | ) * 100
155 |
156 | logger.info(
157 | f"📊 Cache Stats - "
158 | f"Hit Rate: {cache_hit_rate:.1f}% | "
159 | f"Storage: {_CACHE_STATS['storage_hits']}H/{_CACHE_STATS['storage_misses']}M | "
160 | f"Service: {_CACHE_STATS['service_hits']}H/{_CACHE_STATS['service_misses']}M | "
161 | f"Total Time: {total_time:.1f}ms | "
162 | f"Cache Size: {len(_STORAGE_CACHE)} storage + {len(_MEMORY_SERVICE_CACHE)} services"
163 | )
164 |
165 | @dataclass
166 | class MCPServerContext:
167 | """Application context for the MCP server with all required components."""
168 | storage: MemoryStorage
169 | memory_service: MemoryService
170 |
171 | @asynccontextmanager
172 | async def mcp_server_lifespan(server: FastMCP) -> AsyncIterator[MCPServerContext]:
173 | """
174 | Manage MCP server lifecycle with global caching for performance optimization.
175 |
176 | Performance Impact:
177 | - Cache HIT: ~200-400ms (reuses existing instances)
178 | - Cache MISS: ~1,810ms (initializes new instances)
179 |
180 | Caching Strategy:
181 | - Storage instances cached by "{backend}:{path}" key
182 | - MemoryService instances cached by storage ID
183 | - Thread-safe with asyncio.Lock
184 | - Persists across stateless HTTP calls (by design)
185 | """
186 | global _STORAGE_CACHE, _MEMORY_SERVICE_CACHE, _CACHE_STATS
187 |
188 | # Track call statistics
189 | _CACHE_STATS["total_calls"] += 1
190 | start_time = time.time()
191 |
192 | logger.info(f"🔄 MCP Server Call #{_CACHE_STATS['total_calls']} - Checking global cache...")
193 |
194 | # Acquire lock for thread-safe cache access
195 | cache_lock = _get_cache_lock()
196 | async with cache_lock:
197 | # Generate cache key for storage backend
198 | cache_key = f"{STORAGE_BACKEND}:{SQLITE_VEC_PATH}"
199 |
200 | # Check storage cache
201 | if cache_key in _STORAGE_CACHE:
202 | storage = _STORAGE_CACHE[cache_key]
203 | _CACHE_STATS["storage_hits"] += 1
204 | logger.info(f"✅ Storage Cache HIT - Reusing {STORAGE_BACKEND} instance (key: {cache_key})")
205 | else:
206 | _CACHE_STATS["storage_misses"] += 1
207 | logger.info(f"❌ Storage Cache MISS - Initializing {STORAGE_BACKEND} instance...")
208 |
209 | # Initialize storage backend using shared factory
210 | from .storage.factory import create_storage_instance
211 | storage = await create_storage_instance(SQLITE_VEC_PATH, server_type="mcp")
212 |
213 | # Cache the storage instance
214 | _STORAGE_CACHE[cache_key] = storage
215 | init_time = (time.time() - start_time) * 1000 # Convert to ms
216 | _CACHE_STATS["initialization_times"].append(init_time)
217 | logger.info(f"💾 Cached storage instance (key: {cache_key}, init_time: {init_time:.1f}ms)")
218 |
219 | # Check memory service cache and log performance
220 | memory_service = _get_or_create_memory_service(storage)
221 | _log_cache_performance(start_time)
222 |
223 | try:
224 | yield MCPServerContext(
225 | storage=storage,
226 | memory_service=memory_service
227 | )
228 | finally:
229 | # IMPORTANT: Do NOT close cached storage instances here!
230 | # They are intentionally kept alive across stateless HTTP calls for performance.
231 | # Cleanup only happens on process shutdown (handled by FastMCP framework).
232 | logger.info(f"✅ MCP Server Call #{_CACHE_STATS['total_calls']} complete - Cached instances preserved")
233 |
234 | # Create FastMCP server instance
235 | try:
236 | mcp = FastMCP(
237 | name="MCP Memory Service",
238 | host="0.0.0.0", # Listen on all interfaces for remote access
239 | port=8000, # Default port
240 | lifespan=mcp_server_lifespan,
241 | stateless_http=True # Enable stateless HTTP for Claude Code compatibility
242 | )
243 | except TypeError:
244 | # FastMCP not available - create dummy instance
245 | mcp = _DummyFastMCP() # type: ignore
246 |
247 | # =============================================================================
248 | # TYPE DEFINITIONS
249 | # =============================================================================
250 |
251 | class StoreMemorySuccess(TypedDict):
252 | """Return type for successful single memory storage."""
253 | success: bool
254 | message: str
255 | content_hash: str
256 |
257 | class StoreMemorySplitSuccess(TypedDict):
258 | """Return type for successful chunked memory storage."""
259 | success: bool
260 | message: str
261 | chunks_created: int
262 | chunk_hashes: List[str]
263 |
264 | class StoreMemoryFailure(TypedDict):
265 | """Return type for failed memory storage."""
266 | success: bool
267 | message: str
268 | chunks_created: NotRequired[int]
269 | chunk_hashes: NotRequired[List[str]]
270 |
271 | # =============================================================================
272 | # CORE MEMORY OPERATIONS
273 | # =============================================================================
274 |
275 | @mcp.tool()
276 | async def store_memory(
277 | content: str,
278 | ctx: Context,
279 | tags: Union[str, List[str], None] = None,
280 | memory_type: str = "note",
281 | metadata: Optional[Dict[str, Any]] = None,
282 | client_hostname: Optional[str] = None
283 | ) -> Union[StoreMemorySuccess, StoreMemorySplitSuccess, StoreMemoryFailure]:
284 | """
285 | Store a new memory with content and optional metadata.
286 |
287 | **IMPORTANT - Content Length Limits:**
288 | - Cloudflare backend: 800 characters max (BGE model 512 token limit)
289 | - SQLite-vec backend: No limit (local storage)
290 | - Hybrid backend: 800 characters max (constrained by Cloudflare sync)
291 |
292 | If content exceeds the backend's limit, it will be automatically split into
293 | multiple linked memory chunks with preserved context (50-char overlap).
294 | The splitting respects natural boundaries: paragraphs → sentences → words.
295 |
296 | Args:
297 | content: The content to store as memory
298 | tags: Optional tags to categorize the memory (accepts array or comma-separated string)
299 | memory_type: Type of memory (note, decision, task, reference)
300 | metadata: Additional metadata for the memory
301 | client_hostname: Client machine hostname for source tracking
302 |
303 | **Tag Formats - All Formats Supported:**
304 | Both the tags parameter AND metadata.tags accept ALL formats:
305 | - ✅ Array format: tags=["tag1", "tag2", "tag3"]
306 | - ✅ Comma-separated string: tags="tag1,tag2,tag3"
307 | - ✅ Single string: tags="single-tag"
308 | - ✅ In metadata: metadata={"tags": "tag1,tag2", "type": "note"}
309 | - ✅ In metadata (array): metadata={"tags": ["tag1", "tag2"], "type": "note"}
310 |
311 | All formats are automatically normalized internally. If tags are provided in both
312 | the tags parameter and metadata.tags, they will be merged (duplicates removed).
313 |
314 | Returns:
315 | Dictionary with:
316 | - success: Boolean indicating if storage succeeded
317 | - message: Status message
318 | - content_hash: Hash of original content (for single memory)
319 | - chunks_created: Number of chunks (if content was split)
320 | - chunk_hashes: List of content hashes (if content was split)
321 | """
322 | # Delegate to shared MemoryService business logic
323 | memory_service = ctx.request_context.lifespan_context.memory_service
324 | result = await memory_service.store_memory(
325 | content=content,
326 | tags=tags,
327 | memory_type=memory_type,
328 | metadata=metadata,
329 | client_hostname=client_hostname
330 | )
331 |
332 | # Transform MemoryService response to MCP tool format
333 | if not result.get("success"):
334 | return StoreMemoryFailure(
335 | success=False,
336 | message=result.get("error", "Failed to store memory")
337 | )
338 |
339 | # Handle chunked response (multiple memories)
340 | if "memories" in result:
341 | chunk_hashes = [mem["content_hash"] for mem in result["memories"]]
342 | return StoreMemorySplitSuccess(
343 | success=True,
344 | message=f"Successfully stored {len(result['memories'])} memory chunks",
345 | chunks_created=result["total_chunks"],
346 | chunk_hashes=chunk_hashes
347 | )
348 |
349 | # Handle single memory response
350 | memory_data = result["memory"]
351 | return StoreMemorySuccess(
352 | success=True,
353 | message="Memory stored successfully",
354 | content_hash=memory_data["content_hash"]
355 | )
356 |
357 | @mcp.tool()
358 | async def retrieve_memory(
359 | query: str,
360 | ctx: Context,
361 | n_results: int = 5
362 | ) -> Dict[str, Any]:
363 | """
364 | Retrieve memories based on semantic similarity to a query.
365 |
366 | Args:
367 | query: Search query for semantic similarity
368 | n_results: Maximum number of results to return
369 |
370 | Returns:
371 | Dictionary with retrieved memories and metadata
372 | """
373 | # Delegate to shared MemoryService business logic
374 | memory_service = ctx.request_context.lifespan_context.memory_service
375 | return await memory_service.retrieve_memories(
376 | query=query,
377 | n_results=n_results
378 | )
379 |
380 | @mcp.tool()
381 | async def search_by_tag(
382 | tags: Union[str, List[str]],
383 | ctx: Context,
384 | match_all: bool = False
385 | ) -> Dict[str, Any]:
386 | """
387 | Search memories by tags.
388 |
389 | Args:
390 | tags: Tag or list of tags to search for
391 | match_all: If True, memory must have ALL tags; if False, ANY tag
392 |
393 | Returns:
394 | Dictionary with matching memories
395 | """
396 | # Delegate to shared MemoryService business logic
397 | memory_service = ctx.request_context.lifespan_context.memory_service
398 | return await memory_service.search_by_tag(
399 | tags=tags,
400 | match_all=match_all
401 | )
402 |
403 | @mcp.tool()
404 | async def delete_memory(
405 | content_hash: str,
406 | ctx: Context
407 | ) -> Dict[str, Union[bool, str]]:
408 | """
409 | Delete a specific memory by its content hash.
410 |
411 | Args:
412 | content_hash: Hash of the memory content to delete
413 |
414 | Returns:
415 | Dictionary with success status and message
416 | """
417 | # Delegate to shared MemoryService business logic
418 | memory_service = ctx.request_context.lifespan_context.memory_service
419 | return await memory_service.delete_memory(content_hash)
420 |
421 | @mcp.tool()
422 | async def check_database_health(ctx: Context) -> Dict[str, Any]:
423 | """
424 | Check the health and status of the memory database.
425 |
426 | Returns:
427 | Dictionary with health status and statistics
428 | """
429 | # Delegate to shared MemoryService business logic
430 | memory_service = ctx.request_context.lifespan_context.memory_service
431 | return await memory_service.check_database_health()
432 |
433 | @mcp.tool()
434 | async def list_memories(
435 | ctx: Context,
436 | page: int = 1,
437 | page_size: int = 10,
438 | tag: Optional[str] = None,
439 | memory_type: Optional[str] = None
440 | ) -> Dict[str, Any]:
441 | """
442 | List memories with pagination and optional filtering.
443 |
444 | Args:
445 | page: Page number (1-based)
446 | page_size: Number of memories per page
447 | tag: Filter by specific tag
448 | memory_type: Filter by memory type
449 |
450 | Returns:
451 | Dictionary with memories and pagination info
452 | """
453 | # Delegate to shared MemoryService business logic
454 | memory_service = ctx.request_context.lifespan_context.memory_service
455 | return await memory_service.list_memories(
456 | page=page,
457 | page_size=page_size,
458 | tag=tag,
459 | memory_type=memory_type
460 | )
461 |
462 | @mcp.tool()
463 | async def get_cache_stats(ctx: Context) -> Dict[str, Any]:
464 | """
465 | Get MCP server global cache statistics for performance monitoring.
466 |
467 | Returns detailed metrics about storage and memory service caching,
468 | including hit rates, initialization times, and cache sizes.
469 |
470 | This tool is useful for:
471 | - Monitoring cache effectiveness
472 | - Debugging performance issues
473 | - Verifying cache persistence across stateless HTTP calls
474 |
475 | Returns:
476 | Dictionary with cache statistics:
477 | - total_calls: Total MCP server invocations
478 | - hit_rate: Overall cache hit rate percentage
479 | - storage_cache: Storage cache metrics (hits/misses/size)
480 | - service_cache: MemoryService cache metrics (hits/misses/size)
481 | - performance: Initialization time statistics (avg/min/max)
482 | - backend_info: Current storage backend configuration
483 | """
484 | global _CACHE_STATS, _STORAGE_CACHE, _MEMORY_SERVICE_CACHE
485 |
486 | # Import shared stats calculation utility
487 | from mcp_memory_service.utils.cache_manager import CacheStats, calculate_cache_stats_dict
488 |
489 | # Convert global dict to CacheStats dataclass
490 | stats = CacheStats(
491 | total_calls=_CACHE_STATS["total_calls"],
492 | storage_hits=_CACHE_STATS["storage_hits"],
493 | storage_misses=_CACHE_STATS["storage_misses"],
494 | service_hits=_CACHE_STATS["service_hits"],
495 | service_misses=_CACHE_STATS["service_misses"],
496 | initialization_times=_CACHE_STATS["initialization_times"]
497 | )
498 |
499 | # Calculate statistics using shared utility
500 | cache_sizes = (len(_STORAGE_CACHE), len(_MEMORY_SERVICE_CACHE))
501 | result = calculate_cache_stats_dict(stats, cache_sizes)
502 |
503 | # Add server-specific details
504 | result["storage_cache"]["keys"] = list(_STORAGE_CACHE.keys())
505 | result["backend_info"]["embedding_model"] = EMBEDDING_MODEL_NAME
506 |
507 | return result
508 |
509 |
510 |
511 | # =============================================================================
512 | # MAIN ENTRY POINT
513 | # =============================================================================
514 |
515 | def main():
516 | """Main entry point for the FastAPI MCP server."""
517 | # Configure for Claude Code integration
518 | port = int(os.getenv("MCP_SERVER_PORT", "8000"))
519 | host = os.getenv("MCP_SERVER_HOST", "0.0.0.0")
520 |
521 | logger.info(f"Starting MCP Memory Service FastAPI server on {host}:{port}")
522 | logger.info(f"Storage backend: {STORAGE_BACKEND}")
523 |
524 | # Run server with streamable HTTP transport
525 | mcp.run("streamable-http")
526 |
527 | if __name__ == "__main__":
528 | main()
```
--------------------------------------------------------------------------------
/tests/consolidation/test_compression.py:
--------------------------------------------------------------------------------
```python
1 | """Unit tests for the semantic compression engine."""
2 |
3 | import pytest
4 | from datetime import datetime, timedelta
5 |
6 | from mcp_memory_service.consolidation.compression import (
7 | SemanticCompressionEngine,
8 | CompressionResult
9 | )
10 | from mcp_memory_service.consolidation.base import MemoryCluster
11 | from mcp_memory_service.models.memory import Memory
12 |
13 |
14 | @pytest.mark.unit
15 | class TestSemanticCompressionEngine:
16 | """Test the semantic compression system."""
17 |
18 | @pytest.fixture
19 | def compression_engine(self, consolidation_config):
20 | return SemanticCompressionEngine(consolidation_config)
21 |
22 | @pytest.fixture
23 | def sample_cluster_with_memories(self):
24 | """Create a sample cluster with corresponding memories."""
25 | base_time = datetime.now().timestamp()
26 |
27 | memories = [
28 | Memory(
29 | content="Python list comprehensions provide a concise way to create lists",
30 | content_hash="hash1",
31 | tags=["python", "programming", "lists"],
32 | memory_type="reference",
33 | embedding=[0.1, 0.2, 0.3] * 107, # ~320 dim
34 | created_at=base_time - 86400,
35 | created_at_iso=datetime.fromtimestamp(base_time - 86400).isoformat() + 'Z'
36 | ),
37 | Memory(
38 | content="List comprehensions in Python are more readable than traditional for loops",
39 | content_hash="hash2",
40 | tags=["python", "readability", "best-practices"],
41 | memory_type="standard",
42 | embedding=[0.12, 0.18, 0.32] * 107,
43 | created_at=base_time - 172800,
44 | created_at_iso=datetime.fromtimestamp(base_time - 172800).isoformat() + 'Z'
45 | ),
46 | Memory(
47 | content="Example: squares = [x**2 for x in range(10)] creates a list of squares",
48 | content_hash="hash3",
49 | tags=["python", "example", "code"],
50 | memory_type="standard",
51 | embedding=[0.11, 0.21, 0.31] * 107,
52 | created_at=base_time - 259200,
53 | created_at_iso=datetime.fromtimestamp(base_time - 259200).isoformat() + 'Z'
54 | ),
55 | Memory(
56 | content="Python comprehensions work for lists, sets, and dictionaries",
57 | content_hash="hash4",
58 | tags=["python", "comprehensions", "data-structures"],
59 | memory_type="reference",
60 | embedding=[0.13, 0.19, 0.29] * 107,
61 | created_at=base_time - 345600,
62 | created_at_iso=datetime.fromtimestamp(base_time - 345600).isoformat() + 'Z'
63 | )
64 | ]
65 |
66 | cluster = MemoryCluster(
67 | cluster_id="test_cluster",
68 | memory_hashes=[m.content_hash for m in memories],
69 | centroid_embedding=[0.12, 0.2, 0.3] * 107,
70 | coherence_score=0.85,
71 | created_at=datetime.now(),
72 | theme_keywords=["python", "comprehensions", "lists", "programming"],
73 | metadata={"test_cluster": True}
74 | )
75 |
76 | return cluster, memories
77 |
78 | @pytest.mark.asyncio
79 | async def test_basic_compression(self, compression_engine, sample_cluster_with_memories):
80 | """Test basic compression functionality."""
81 | cluster, memories = sample_cluster_with_memories
82 |
83 | results = await compression_engine.process([cluster], memories)
84 |
85 | assert len(results) == 1
86 | result = results[0]
87 |
88 | assert isinstance(result, CompressionResult)
89 | assert result.cluster_id == "test_cluster"
90 | assert isinstance(result.compressed_memory, Memory)
91 | assert result.source_memory_count == 4
92 | assert 0 < result.compression_ratio < 1 # Should be compressed
93 | assert len(result.key_concepts) > 0
94 | assert isinstance(result.temporal_span, dict)
95 |
96 | @pytest.mark.asyncio
97 | async def test_compressed_memory_properties(self, compression_engine, sample_cluster_with_memories):
98 | """Test properties of the compressed memory object."""
99 | cluster, memories = sample_cluster_with_memories
100 |
101 | results = await compression_engine.process([cluster], memories)
102 | compressed_memory = results[0].compressed_memory
103 |
104 | # Check basic properties
105 | assert compressed_memory.memory_type == "compressed_cluster"
106 | assert len(compressed_memory.content) <= compression_engine.max_summary_length
107 | assert len(compressed_memory.content) > 0
108 | assert compressed_memory.content_hash is not None
109 |
110 | # Check tags (should include cluster tags and compression marker)
111 | assert "compressed_cluster" in compressed_memory.tags or "compressed" in compressed_memory.tags
112 |
113 | # Check metadata
114 | assert "cluster_id" in compressed_memory.metadata
115 | assert "compression_date" in compressed_memory.metadata
116 | assert "source_memory_count" in compressed_memory.metadata
117 | assert "compression_ratio" in compressed_memory.metadata
118 | assert "key_concepts" in compressed_memory.metadata
119 | assert "temporal_span" in compressed_memory.metadata
120 | assert "theme_keywords" in compressed_memory.metadata
121 |
122 | # Check embedding (should use cluster centroid)
123 | assert compressed_memory.embedding == cluster.centroid_embedding
124 |
125 | @pytest.mark.asyncio
126 | async def test_key_concept_extraction(self, compression_engine, sample_cluster_with_memories):
127 | """Test extraction of key concepts from cluster memories."""
128 | cluster, memories = sample_cluster_with_memories
129 |
130 | key_concepts = await compression_engine._extract_key_concepts(memories, cluster.theme_keywords)
131 |
132 | assert isinstance(key_concepts, list)
133 | assert len(key_concepts) > 0
134 |
135 | # Should include theme keywords
136 | theme_overlap = set(key_concepts).intersection(set(cluster.theme_keywords))
137 | assert len(theme_overlap) > 0
138 |
139 | # Should extract relevant concepts from content
140 | expected_concepts = {"python", "comprehensions", "lists"}
141 | found_concepts = set(concept.lower() for concept in key_concepts)
142 | overlap = expected_concepts.intersection(found_concepts)
143 | assert len(overlap) > 0
144 |
145 | @pytest.mark.asyncio
146 | async def test_thematic_summary_generation(self, compression_engine, sample_cluster_with_memories):
147 | """Test generation of thematic summaries."""
148 | cluster, memories = sample_cluster_with_memories
149 |
150 | # Extract key concepts first
151 | key_concepts = await compression_engine._extract_key_concepts(memories, cluster.theme_keywords)
152 |
153 | # Generate summary
154 | summary = await compression_engine._generate_thematic_summary(memories, key_concepts)
155 |
156 | assert isinstance(summary, str)
157 | assert len(summary) > 0
158 | assert len(summary) <= compression_engine.max_summary_length
159 |
160 | # Summary should contain information about the cluster
161 | summary_lower = summary.lower()
162 | assert "cluster" in summary_lower or str(len(memories)) in summary
163 |
164 | # Should mention key concepts
165 | concept_mentions = sum(1 for concept in key_concepts[:3] if concept.lower() in summary_lower)
166 | assert concept_mentions > 0
167 |
168 | @pytest.mark.asyncio
169 | async def test_temporal_span_calculation(self, compression_engine, sample_cluster_with_memories):
170 | """Test calculation of temporal span for memories."""
171 | cluster, memories = sample_cluster_with_memories
172 |
173 | temporal_span = compression_engine._calculate_temporal_span(memories)
174 |
175 | assert isinstance(temporal_span, dict)
176 | assert "start_time" in temporal_span
177 | assert "end_time" in temporal_span
178 | assert "span_days" in temporal_span
179 | assert "span_description" in temporal_span
180 | assert "start_iso" in temporal_span
181 | assert "end_iso" in temporal_span
182 |
183 | # Check values make sense
184 | assert temporal_span["start_time"] <= temporal_span["end_time"]
185 | assert temporal_span["span_days"] >= 0
186 | assert isinstance(temporal_span["span_description"], str)
187 |
188 | @pytest.mark.asyncio
189 | async def test_tag_aggregation(self, compression_engine, sample_cluster_with_memories):
190 | """Test aggregation of tags from cluster memories."""
191 | cluster, memories = sample_cluster_with_memories
192 |
193 | aggregated_tags = compression_engine._aggregate_tags(memories)
194 |
195 | assert isinstance(aggregated_tags, list)
196 | assert "cluster" in aggregated_tags
197 | assert "compressed" in aggregated_tags
198 |
199 | # Should include frequent tags from original memories
200 | original_tags = set()
201 | for memory in memories:
202 | original_tags.update(memory.tags)
203 |
204 | # Check that some original tags are preserved
205 | aggregated_set = set(aggregated_tags)
206 | overlap = original_tags.intersection(aggregated_set)
207 | assert len(overlap) > 0
208 |
209 | @pytest.mark.asyncio
210 | async def test_metadata_aggregation(self, compression_engine, sample_cluster_with_memories):
211 | """Test aggregation of metadata from cluster memories."""
212 | cluster, memories = sample_cluster_with_memories
213 |
214 | # Add some metadata to memories
215 | memories[0].metadata["test_field"] = "value1"
216 | memories[1].metadata["test_field"] = "value1" # Same value
217 | memories[2].metadata["test_field"] = "value2" # Different value
218 | memories[3].metadata["unique_field"] = "unique"
219 |
220 | aggregated_metadata = compression_engine._aggregate_metadata(memories)
221 |
222 | assert isinstance(aggregated_metadata, dict)
223 | assert "source_memory_hashes" in aggregated_metadata
224 |
225 | # Should handle common values
226 | if "common_test_field" in aggregated_metadata:
227 | assert aggregated_metadata["common_test_field"] in ["value1", "value2"]
228 |
229 | # Should handle varied values
230 | if "varied_test_field" in aggregated_metadata:
231 | assert isinstance(aggregated_metadata["varied_test_field"], list)
232 |
233 | # Should track variety
234 | if "unique_field_variety_count" in aggregated_metadata:
235 | assert aggregated_metadata["unique_field_variety_count"] == 1
236 |
237 | @pytest.mark.asyncio
238 | async def test_compression_ratio_calculation(self, compression_engine, sample_cluster_with_memories):
239 | """Test compression ratio calculation."""
240 | cluster, memories = sample_cluster_with_memories
241 |
242 | results = await compression_engine.process([cluster], memories)
243 | result = results[0]
244 |
245 | # Calculate expected ratio
246 | original_size = sum(len(m.content) for m in memories)
247 | compressed_size = len(result.compressed_memory.content)
248 | expected_ratio = compressed_size / original_size
249 |
250 | assert abs(result.compression_ratio - expected_ratio) < 0.01 # Small tolerance
251 | assert 0 < result.compression_ratio < 1 # Should be compressed
252 |
253 | @pytest.mark.asyncio
254 | async def test_sentence_splitting(self, compression_engine):
255 | """Test sentence splitting functionality."""
256 | text = "This is the first sentence. This is the second sentence! Is this a question? Yes, it is."
257 |
258 | sentences = compression_engine._split_into_sentences(text)
259 |
260 | assert isinstance(sentences, list)
261 | assert len(sentences) >= 3 # Should find multiple sentences
262 |
263 | # Check that sentences are properly cleaned
264 | for sentence in sentences:
265 | assert len(sentence) > 10 # Minimum length filter
266 | assert sentence.strip() == sentence # Should be trimmed
267 |
268 | @pytest.mark.asyncio
269 | async def test_empty_cluster_handling(self, compression_engine):
270 | """Test handling of empty clusters."""
271 | results = await compression_engine.process([], [])
272 | assert results == []
273 |
274 | @pytest.mark.asyncio
275 | async def test_single_memory_cluster(self, compression_engine):
276 | """Test handling of cluster with single memory (should be skipped)."""
277 | memory = Memory(
278 | content="Single memory content",
279 | content_hash="single",
280 | tags=["test"],
281 | embedding=[0.1] * 320,
282 | created_at=datetime.now().timestamp()
283 | )
284 |
285 | cluster = MemoryCluster(
286 | cluster_id="single_cluster",
287 | memory_hashes=["single"],
288 | centroid_embedding=[0.1] * 320,
289 | coherence_score=1.0,
290 | created_at=datetime.now(),
291 | theme_keywords=["test"]
292 | )
293 |
294 | results = await compression_engine.process([cluster], [memory])
295 |
296 | # Should skip clusters with insufficient memories
297 | assert results == []
298 |
299 | @pytest.mark.asyncio
300 | async def test_missing_memories_handling(self, compression_engine):
301 | """Test handling of cluster referencing missing memories."""
302 | cluster = MemoryCluster(
303 | cluster_id="missing_cluster",
304 | memory_hashes=["missing1", "missing2", "missing3"],
305 | centroid_embedding=[0.1] * 320,
306 | coherence_score=0.8,
307 | created_at=datetime.now(),
308 | theme_keywords=["missing"]
309 | )
310 |
311 | # Provide empty memories list
312 | results = await compression_engine.process([cluster], [])
313 |
314 | # Should handle missing memories gracefully
315 | assert results == []
316 |
317 | @pytest.mark.asyncio
318 | async def test_compression_benefit_estimation(self, compression_engine, sample_cluster_with_memories):
319 | """Test estimation of compression benefits."""
320 | cluster, memories = sample_cluster_with_memories
321 |
322 | benefits = await compression_engine.estimate_compression_benefit([cluster], memories)
323 |
324 | assert isinstance(benefits, dict)
325 | assert "compressible_clusters" in benefits
326 | assert "total_original_size" in benefits
327 | assert "estimated_compressed_size" in benefits
328 | assert "compression_ratio" in benefits
329 | assert "estimated_savings_bytes" in benefits
330 | assert "estimated_savings_percent" in benefits
331 |
332 | # Check values make sense
333 | assert benefits["compressible_clusters"] >= 0
334 | assert benefits["total_original_size"] >= 0
335 | assert benefits["estimated_compressed_size"] >= 0
336 | assert 0 <= benefits["compression_ratio"] <= 1
337 | assert benefits["estimated_savings_bytes"] >= 0
338 | assert 0 <= benefits["estimated_savings_percent"] <= 100
339 |
340 | @pytest.mark.asyncio
341 | async def test_large_content_truncation(self, compression_engine):
342 | """Test handling of content that exceeds max summary length."""
343 | # Create memories with very long content
344 | long_memories = []
345 | base_time = datetime.now().timestamp()
346 |
347 | for i in range(3):
348 | # Create content longer than max_summary_length
349 | long_content = "This is a very long memory content. " * 50 # Much longer than 200 chars
350 | memory = Memory(
351 | content=long_content,
352 | content_hash=f"long_{i}",
353 | tags=["long", "test"],
354 | embedding=[0.1 + i*0.1] * 320,
355 | created_at=base_time - (i * 3600)
356 | )
357 | long_memories.append(memory)
358 |
359 | cluster = MemoryCluster(
360 | cluster_id="long_cluster",
361 | memory_hashes=[m.content_hash for m in long_memories],
362 | centroid_embedding=[0.2] * 320,
363 | coherence_score=0.8,
364 | created_at=datetime.now(),
365 | theme_keywords=["long", "content"]
366 | )
367 |
368 | results = await compression_engine.process([cluster], long_memories)
369 |
370 | if results:
371 | compressed_content = results[0].compressed_memory.content
372 | # Should be truncated to max length
373 | assert len(compressed_content) <= compression_engine.max_summary_length
374 |
375 | # Should indicate truncation if content was cut off
376 | if len(compressed_content) == compression_engine.max_summary_length:
377 | assert compressed_content.endswith("...")
378 |
379 | @pytest.mark.asyncio
380 | async def test_key_concept_extraction_comprehensive(self, compression_engine):
381 | """Test comprehensive key concept extraction from memories."""
382 | # Create memories with various content patterns
383 | memories = []
384 | base_time = datetime.now().timestamp()
385 |
386 | content_examples = [
387 | "Check out https://example.com for more info about CamelCaseVariable usage.",
388 | "Email me at [email protected] if you have questions about the API response.",
389 | "The system returns {'status': 'success', 'code': 200} for valid requests.",
390 | "Today's date is 2024-01-15 and the time is 14:30 for scheduling.",
391 | "See 'important documentation' for details on snake_case_variable patterns."
392 | ]
393 |
394 | for i, content in enumerate(content_examples):
395 | memory = Memory(
396 | content=content,
397 | content_hash=f"concept_test_{i}",
398 | tags=["test", "concept", "extraction"],
399 | embedding=[0.1 + i*0.01] * 320,
400 | created_at=base_time - (i * 3600)
401 | )
402 | memories.append(memory)
403 |
404 | theme_keywords = ["test", "API", "documentation", "variable"]
405 |
406 | concepts = await compression_engine._extract_key_concepts(memories, theme_keywords)
407 |
408 | # Should include theme keywords
409 | assert any("test" in concepts for concept in [theme_keywords])
410 |
411 | # Should extract concepts from content
412 | assert isinstance(concepts, list)
413 | assert len(concepts) > 0
414 |
415 | # Concepts should be strings
416 | assert all(isinstance(concept, str) for concept in concepts)
417 |
418 | @pytest.mark.asyncio
419 | async def test_memories_without_timestamps(self, compression_engine):
420 | """Test handling of memories with timestamps (Memory model auto-sets them)."""
421 | memories = [
422 | Memory(
423 | content="Memory with auto-generated timestamp",
424 | content_hash="auto_timestamp",
425 | tags=["test"],
426 | embedding=[0.1] * 320,
427 | created_at=None # Will be auto-set by Memory model
428 | )
429 | ]
430 |
431 | cluster = MemoryCluster(
432 | cluster_id="auto_timestamp_cluster",
433 | memory_hashes=["auto_timestamp"],
434 | centroid_embedding=[0.1] * 320,
435 | coherence_score=0.8,
436 | created_at=datetime.now(),
437 | theme_keywords=["test"]
438 | )
439 |
440 | # Should handle gracefully without crashing
441 | temporal_span = compression_engine._calculate_temporal_span(memories)
442 |
443 | # Memory model auto-sets timestamps, so these will be actual values
444 | assert temporal_span["start_time"] is not None
445 | assert temporal_span["end_time"] is not None
446 | assert temporal_span["span_days"] >= 0
447 | assert isinstance(temporal_span["span_description"], str)
```
--------------------------------------------------------------------------------
/tests/test_hybrid_storage.py:
--------------------------------------------------------------------------------
```python
1 | #!/usr/bin/env python3
2 | """
3 | Comprehensive tests for HybridMemoryStorage implementation.
4 |
5 | Tests cover:
6 | - Basic storage operations (store, retrieve, delete)
7 | - Background synchronization service
8 | - Failover and graceful degradation
9 | - Configuration and health monitoring
10 | - Performance characteristics
11 | """
12 |
13 | import asyncio
14 | import pytest
15 | import pytest_asyncio
16 | import tempfile
17 | import os
18 | import sys
19 | import logging
20 | from pathlib import Path
21 | from unittest.mock import AsyncMock, MagicMock, patch
22 | from typing import Dict, Any
23 |
24 | # Add src to path for imports
25 | current_dir = Path(__file__).parent
26 | src_dir = current_dir.parent / "src"
27 | sys.path.insert(0, str(src_dir))
28 |
29 | from mcp_memory_service.storage.hybrid import HybridMemoryStorage, BackgroundSyncService, SyncOperation
30 | from mcp_memory_service.storage.sqlite_vec import SqliteVecMemoryStorage
31 | from mcp_memory_service.models.memory import Memory, MemoryQueryResult
32 | from mcp_memory_service.utils.hashing import generate_content_hash
33 |
34 | # Configure test logging
35 | logging.basicConfig(level=logging.DEBUG)
36 | logger = logging.getLogger(__name__)
37 |
38 | class MockCloudflareStorage:
39 | """Mock Cloudflare storage for testing."""
40 |
41 | def __init__(self, **kwargs):
42 | self.initialized = False
43 | self.stored_memories = {}
44 | self.fail_operations = False
45 | self.fail_initialization = False
46 |
47 | async def initialize(self):
48 | if self.fail_initialization:
49 | raise Exception("Mock Cloudflare initialization failed")
50 | self.initialized = True
51 |
52 | async def store(self, memory: Memory):
53 | if self.fail_operations:
54 | return False, "Mock Cloudflare operation failed"
55 | self.stored_memories[memory.content_hash] = memory
56 | return True, "Memory stored successfully"
57 |
58 | async def delete(self, content_hash: str):
59 | if self.fail_operations:
60 | return False, "Mock Cloudflare operation failed"
61 | if content_hash in self.stored_memories:
62 | del self.stored_memories[content_hash]
63 | return True, "Memory deleted successfully"
64 | return False, "Memory not found"
65 |
66 | async def update_memory_metadata(self, content_hash: str, updates: Dict[str, Any], preserve_timestamps: bool = True):
67 | if self.fail_operations:
68 | return False, "Mock Cloudflare operation failed"
69 | if content_hash in self.stored_memories:
70 | # Simple mock update
71 | return True, "Memory updated successfully"
72 | return False, "Memory not found"
73 |
74 | async def get_stats(self):
75 | if self.fail_operations:
76 | raise Exception("Mock Cloudflare stats failed")
77 | return {
78 | "total_memories": len(self.stored_memories),
79 | "storage_backend": "MockCloudflareStorage"
80 | }
81 |
82 | async def close(self):
83 | pass
84 |
85 |
86 | @pytest_asyncio.fixture
87 | async def temp_sqlite_db():
88 | """Create a temporary SQLite database for testing."""
89 | with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as tmp_file:
90 | db_path = tmp_file.name
91 |
92 | yield db_path
93 |
94 | # Cleanup
95 | if os.path.exists(db_path):
96 | os.unlink(db_path)
97 |
98 |
99 | @pytest_asyncio.fixture
100 | async def mock_cloudflare_config():
101 | """Mock Cloudflare configuration for testing."""
102 | return {
103 | 'api_token': 'test_token',
104 | 'account_id': 'test_account',
105 | 'vectorize_index': 'test_index',
106 | 'd1_database_id': 'test_db_id'
107 | }
108 |
109 |
110 | @pytest_asyncio.fixture
111 | async def hybrid_storage(temp_sqlite_db, mock_cloudflare_config):
112 | """Create a HybridMemoryStorage instance for testing."""
113 | with patch('mcp_memory_service.storage.hybrid.CloudflareStorage', MockCloudflareStorage):
114 | storage = HybridMemoryStorage(
115 | sqlite_db_path=temp_sqlite_db,
116 | embedding_model="all-MiniLM-L6-v2",
117 | cloudflare_config=mock_cloudflare_config,
118 | sync_interval=1, # Short interval for testing
119 | batch_size=5
120 | )
121 | await storage.initialize()
122 | yield storage
123 | await storage.close()
124 |
125 |
126 | @pytest.fixture
127 | def sample_memory():
128 | """Create a sample memory for testing."""
129 | content = "This is a test memory for hybrid storage"
130 | return Memory(
131 | content=content,
132 | content_hash=generate_content_hash(content),
133 | tags=["test", "sample"],
134 | memory_type="test",
135 | metadata={},
136 | created_at=1638360000.0
137 | )
138 |
139 |
140 | class TestHybridMemoryStorage:
141 | """Test cases for HybridMemoryStorage functionality."""
142 |
143 | @pytest.mark.asyncio
144 | async def test_initialization_with_cloudflare(self, temp_sqlite_db, mock_cloudflare_config):
145 | """Test successful initialization with Cloudflare configuration."""
146 | with patch('mcp_memory_service.storage.hybrid.CloudflareStorage', MockCloudflareStorage):
147 | storage = HybridMemoryStorage(
148 | sqlite_db_path=temp_sqlite_db,
149 | cloudflare_config=mock_cloudflare_config
150 | )
151 |
152 | await storage.initialize()
153 |
154 | assert storage.initialized
155 | assert storage.primary is not None
156 | assert storage.secondary is not None
157 | assert storage.sync_service is not None
158 | assert storage.sync_service.is_running
159 |
160 | await storage.close()
161 |
162 | @pytest.mark.asyncio
163 | async def test_initialization_without_cloudflare(self, temp_sqlite_db):
164 | """Test initialization without Cloudflare configuration (SQLite-only mode)."""
165 | storage = HybridMemoryStorage(sqlite_db_path=temp_sqlite_db)
166 |
167 | await storage.initialize()
168 |
169 | assert storage.initialized
170 | assert storage.primary is not None
171 | assert storage.secondary is None
172 | assert storage.sync_service is None
173 |
174 | await storage.close()
175 |
176 | @pytest.mark.asyncio
177 | async def test_initialization_with_cloudflare_failure(self, temp_sqlite_db, mock_cloudflare_config):
178 | """Test graceful handling of Cloudflare initialization failure."""
179 | def failing_cloudflare_storage(**kwargs):
180 | storage = MockCloudflareStorage(**kwargs)
181 | storage.fail_initialization = True
182 | return storage
183 |
184 | with patch('mcp_memory_service.storage.hybrid.CloudflareStorage', failing_cloudflare_storage):
185 | storage = HybridMemoryStorage(
186 | sqlite_db_path=temp_sqlite_db,
187 | cloudflare_config=mock_cloudflare_config
188 | )
189 |
190 | await storage.initialize()
191 |
192 | # Should fall back to SQLite-only mode
193 | assert storage.initialized
194 | assert storage.primary is not None
195 | assert storage.secondary is None
196 | assert storage.sync_service is None
197 |
198 | await storage.close()
199 |
200 | @pytest.mark.asyncio
201 | async def test_store_memory(self, hybrid_storage, sample_memory):
202 | """Test storing a memory in hybrid storage."""
203 | success, message = await hybrid_storage.store(sample_memory)
204 |
205 | assert success
206 | assert "success" in message.lower() or message == ""
207 |
208 | # Verify memory is stored in primary
209 | results = await hybrid_storage.retrieve(sample_memory.content, n_results=1)
210 | assert len(results) == 1
211 | assert results[0].memory.content == sample_memory.content
212 |
213 | @pytest.mark.asyncio
214 | async def test_retrieve_memory(self, hybrid_storage, sample_memory):
215 | """Test retrieving memories from hybrid storage."""
216 | # Store a memory first
217 | await hybrid_storage.store(sample_memory)
218 |
219 | # Retrieve by query
220 | results = await hybrid_storage.retrieve("test memory", n_results=5)
221 | assert len(results) >= 1
222 |
223 | # Check that we get the stored memory
224 | found = any(result.memory.content == sample_memory.content for result in results)
225 | assert found
226 |
227 | @pytest.mark.asyncio
228 | async def test_delete_memory(self, hybrid_storage, sample_memory):
229 | """Test deleting a memory from hybrid storage."""
230 | # Store a memory first
231 | await hybrid_storage.store(sample_memory)
232 |
233 | # Delete the memory
234 | success, message = await hybrid_storage.delete(sample_memory.content_hash)
235 | assert success
236 |
237 | # Verify memory is deleted from primary
238 | results = await hybrid_storage.retrieve(sample_memory.content, n_results=1)
239 | # Should not find the deleted memory
240 | found = any(result.memory.content_hash == sample_memory.content_hash for result in results)
241 | assert not found
242 |
243 | @pytest.mark.asyncio
244 | async def test_search_by_tags(self, hybrid_storage, sample_memory):
245 | """Test searching memories by tags."""
246 | # Store a memory first
247 | await hybrid_storage.store(sample_memory)
248 |
249 | # Search by tags
250 | results = await hybrid_storage.search_by_tags(["test"])
251 | assert len(results) >= 1
252 |
253 | # Check that we get the stored memory
254 | found = any(memory.content == sample_memory.content for memory in results)
255 | assert found
256 |
257 | @pytest.mark.asyncio
258 | async def test_get_stats(self, hybrid_storage):
259 | """Test getting statistics from hybrid storage."""
260 | stats = await hybrid_storage.get_stats()
261 |
262 | assert "storage_backend" in stats
263 | assert stats["storage_backend"] == "Hybrid (SQLite-vec + Cloudflare)"
264 | assert "primary_stats" in stats
265 | assert "sync_enabled" in stats
266 | assert stats["sync_enabled"] == True
267 | assert "sync_status" in stats
268 |
269 | @pytest.mark.asyncio
270 | async def test_force_sync(self, hybrid_storage, sample_memory):
271 | """Test forcing immediate synchronization."""
272 | # Store some memories
273 | await hybrid_storage.store(sample_memory)
274 |
275 | # Force sync
276 | result = await hybrid_storage.force_sync()
277 |
278 | assert "status" in result
279 | assert result["status"] in ["completed", "partial"]
280 | assert "primary_memories" in result
281 | assert result["primary_memories"] >= 1
282 |
283 |
284 | class TestBackgroundSyncService:
285 | """Test cases for BackgroundSyncService functionality."""
286 |
287 | @pytest_asyncio.fixture
288 | async def sync_service_components(self, temp_sqlite_db):
289 | """Create components needed for sync service testing."""
290 | primary = SqliteVecMemoryStorage(temp_sqlite_db)
291 | await primary.initialize()
292 |
293 | secondary = MockCloudflareStorage()
294 | await secondary.initialize()
295 |
296 | sync_service = BackgroundSyncService(
297 | primary, secondary,
298 | sync_interval=1,
299 | batch_size=3
300 | )
301 |
302 | yield primary, secondary, sync_service
303 |
304 | if sync_service.is_running:
305 | await sync_service.stop()
306 |
307 | if hasattr(primary, 'close'):
308 | await primary.close()
309 |
310 | @pytest.mark.asyncio
311 | async def test_sync_service_start_stop(self, sync_service_components):
312 | """Test starting and stopping the background sync service."""
313 | primary, secondary, sync_service = sync_service_components
314 |
315 | # Start service
316 | await sync_service.start()
317 | assert sync_service.is_running
318 |
319 | # Stop service
320 | await sync_service.stop()
321 | assert not sync_service.is_running
322 |
323 | @pytest.mark.asyncio
324 | async def test_operation_enqueue(self, sync_service_components, sample_memory):
325 | """Test enqueuing sync operations."""
326 | primary, secondary, sync_service = sync_service_components
327 |
328 | await sync_service.start()
329 |
330 | # Enqueue a store operation
331 | operation = SyncOperation(operation='store', memory=sample_memory)
332 | await sync_service.enqueue_operation(operation)
333 |
334 | # Wait a bit for processing
335 | await asyncio.sleep(0.1)
336 |
337 | # Check queue size decreased
338 | status = await sync_service.get_sync_status()
339 | assert status['queue_size'] >= 0 # Should be processed or in progress
340 |
341 | await sync_service.stop()
342 |
343 | @pytest.mark.asyncio
344 | async def test_sync_with_cloudflare_failure(self, sync_service_components):
345 | """Test sync behavior when Cloudflare operations fail."""
346 | primary, secondary, sync_service = sync_service_components
347 |
348 | # Make Cloudflare operations fail
349 | secondary.fail_operations = True
350 |
351 | await sync_service.start()
352 |
353 | # Create a test memory
354 | content = "test content"
355 | memory = Memory(
356 | content=content,
357 | content_hash=generate_content_hash(content),
358 | tags=["test"],
359 | memory_type="test"
360 | )
361 |
362 | # Enqueue operation
363 | operation = SyncOperation(operation='store', memory=memory)
364 | await sync_service.enqueue_operation(operation)
365 |
366 | # Wait for processing
367 | await asyncio.sleep(0.2)
368 |
369 | # Check that service marked Cloudflare as unavailable
370 | status = await sync_service.get_sync_status()
371 | assert status['cloudflare_available'] == False
372 |
373 | await sync_service.stop()
374 |
375 | @pytest.mark.asyncio
376 | async def test_force_sync_functionality(self, sync_service_components):
377 | """Test force sync functionality."""
378 | primary, secondary, sync_service = sync_service_components
379 |
380 | # Store some test memories in primary
381 | content1 = "test memory 1"
382 | content2 = "test memory 2"
383 | memory1 = Memory(
384 | content=content1,
385 | content_hash=generate_content_hash(content1),
386 | tags=["test"],
387 | memory_type="test"
388 | )
389 | memory2 = Memory(
390 | content=content2,
391 | content_hash=generate_content_hash(content2),
392 | tags=["test"],
393 | memory_type="test"
394 | )
395 |
396 | await primary.store(memory1)
397 | await primary.store(memory2)
398 |
399 | await sync_service.start()
400 |
401 | # Force sync
402 | result = await sync_service.force_sync()
403 |
404 | assert result['status'] == 'completed'
405 | assert result['primary_memories'] == 2
406 | assert result['synced_to_secondary'] >= 0
407 |
408 | await sync_service.stop()
409 |
410 | @pytest.mark.asyncio
411 | async def test_sync_status_reporting(self, sync_service_components):
412 | """Test sync status reporting functionality."""
413 | primary, secondary, sync_service = sync_service_components
414 |
415 | await sync_service.start()
416 |
417 | status = await sync_service.get_sync_status()
418 |
419 | assert 'is_running' in status
420 | assert status['is_running'] == True
421 | assert 'queue_size' in status
422 | assert 'stats' in status
423 | assert 'cloudflare_available' in status
424 |
425 | await sync_service.stop()
426 |
427 |
428 | class TestPerformanceCharacteristics:
429 | """Test performance characteristics of hybrid storage."""
430 |
431 | @pytest.mark.asyncio
432 | async def test_read_performance(self, hybrid_storage, sample_memory):
433 | """Test that reads are fast (should use SQLite-vec)."""
434 | # Store a memory
435 | await hybrid_storage.store(sample_memory)
436 |
437 | # Measure read performance
438 | import time
439 |
440 | start_time = time.time()
441 | results = await hybrid_storage.retrieve(sample_memory.content[:10], n_results=1)
442 | duration = time.time() - start_time
443 |
444 | # Should be very fast (< 100ms for local SQLite-vec)
445 | assert duration < 0.1
446 | assert len(results) >= 0 # Should get some results
447 |
448 | @pytest.mark.asyncio
449 | async def test_write_performance(self, hybrid_storage):
450 | """Test that writes are fast (immediate SQLite-vec write)."""
451 | content = "Performance test memory"
452 | memory = Memory(
453 | content=content,
454 | content_hash=generate_content_hash(content),
455 | tags=["perf"],
456 | memory_type="performance_test"
457 | )
458 |
459 | import time
460 |
461 | start_time = time.time()
462 | success, message = await hybrid_storage.store(memory)
463 | duration = time.time() - start_time
464 |
465 | # Should be very fast (< 100ms for local SQLite-vec)
466 | assert duration < 0.1
467 | assert success
468 |
469 | @pytest.mark.asyncio
470 | async def test_concurrent_operations(self, hybrid_storage):
471 | """Test concurrent memory operations."""
472 | # Create multiple memories
473 | memories = []
474 | for i in range(10):
475 | content = f"Concurrent test memory {i}"
476 | memory = Memory(
477 | content=content,
478 | content_hash=generate_content_hash(content),
479 | tags=["concurrent", f"test{i}"],
480 | memory_type="concurrent_test"
481 | )
482 | memories.append(memory)
483 |
484 | # Store all memories concurrently
485 | tasks = [hybrid_storage.store(memory) for memory in memories]
486 | results = await asyncio.gather(*tasks)
487 |
488 | # All operations should succeed
489 | assert all(success for success, message in results)
490 |
491 | # Should be able to retrieve all memories
492 | search_results = await hybrid_storage.search_by_tags(["concurrent"])
493 | assert len(search_results) == 10
494 |
495 |
496 | class TestErrorHandlingAndFallback:
497 | """Test error handling and fallback scenarios."""
498 |
499 | @pytest.mark.asyncio
500 | async def test_sqlite_only_mode(self, temp_sqlite_db):
501 | """Test operation in SQLite-only mode (no Cloudflare)."""
502 | storage = HybridMemoryStorage(sqlite_db_path=temp_sqlite_db)
503 | await storage.initialize()
504 |
505 | # Should work normally without Cloudflare
506 | content = "SQLite-only test memory"
507 | memory = Memory(
508 | content=content,
509 | content_hash=generate_content_hash(content),
510 | tags=["local"],
511 | memory_type="sqlite_only"
512 | )
513 |
514 | success, message = await storage.store(memory)
515 | assert success
516 |
517 | results = await storage.retrieve(memory.content, n_results=1)
518 | assert len(results) >= 1
519 |
520 | await storage.close()
521 |
522 | @pytest.mark.asyncio
523 | async def test_graceful_degradation(self, temp_sqlite_db, mock_cloudflare_config):
524 | """Test graceful degradation when Cloudflare becomes unavailable."""
525 | def unreliable_cloudflare_storage(**kwargs):
526 | storage = MockCloudflareStorage(**kwargs)
527 | # Will start working but then fail
528 | return storage
529 |
530 | with patch('mcp_memory_service.storage.hybrid.CloudflareStorage', unreliable_cloudflare_storage):
531 | storage = HybridMemoryStorage(
532 | sqlite_db_path=temp_sqlite_db,
533 | cloudflare_config=mock_cloudflare_config
534 | )
535 |
536 | await storage.initialize()
537 |
538 | # Initially should work
539 | content = "Degradation test memory"
540 | memory = Memory(
541 | content=content,
542 | content_hash=generate_content_hash(content),
543 | tags=["test"],
544 | memory_type="degradation_test"
545 | )
546 |
547 | success, message = await storage.store(memory)
548 | assert success
549 |
550 | # Make Cloudflare fail
551 | storage.secondary.fail_operations = True
552 |
553 | # Should still work (primary storage unaffected)
554 | content2 = "Second test memory"
555 | memory2 = Memory(
556 | content=content2,
557 | content_hash=generate_content_hash(content2),
558 | tags=["test"],
559 | memory_type="degradation_test"
560 | )
561 | success, message = await storage.store(memory2)
562 | assert success
563 |
564 | # Retrieval should still work
565 | results = await storage.retrieve("test memory", n_results=10)
566 | assert len(results) >= 2
567 |
568 | await storage.close()
569 |
570 |
571 | if __name__ == "__main__":
572 | # Run tests
573 | pytest.main([__file__, "-v"])
```