This is page 15 of 39. Use http://codebase.md/wshobson/maverick-mcp?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .dockerignore
├── .env.example
├── .github
│ ├── dependabot.yml
│ ├── FUNDING.yml
│ ├── ISSUE_TEMPLATE
│ │ ├── bug_report.md
│ │ ├── config.yml
│ │ ├── feature_request.md
│ │ ├── question.md
│ │ └── security_report.md
│ ├── pull_request_template.md
│ └── workflows
│ ├── claude-code-review.yml
│ └── claude.yml
├── .gitignore
├── .python-version
├── .vscode
│ ├── launch.json
│ └── settings.json
├── alembic
│ ├── env.py
│ ├── script.py.mako
│ └── versions
│ ├── 001_initial_schema.py
│ ├── 003_add_performance_indexes.py
│ ├── 006_rename_metadata_columns.py
│ ├── 008_performance_optimization_indexes.py
│ ├── 009_rename_to_supply_demand.py
│ ├── 010_self_contained_schema.py
│ ├── 011_remove_proprietary_terms.py
│ ├── 013_add_backtest_persistence_models.py
│ ├── 014_add_portfolio_models.py
│ ├── 08e3945a0c93_merge_heads.py
│ ├── 9374a5c9b679_merge_heads_for_testing.py
│ ├── abf9b9afb134_merge_multiple_heads.py
│ ├── adda6d3fd84b_merge_proprietary_terms_removal_with_.py
│ ├── e0c75b0bdadb_fix_financial_data_precision_only.py
│ ├── f0696e2cac15_add_essential_performance_indexes.py
│ └── fix_database_integrity_issues.py
├── alembic.ini
├── CLAUDE.md
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── DATABASE_SETUP.md
├── docker-compose.override.yml.example
├── docker-compose.yml
├── Dockerfile
├── docs
│ ├── api
│ │ └── backtesting.md
│ ├── BACKTESTING.md
│ ├── COST_BASIS_SPECIFICATION.md
│ ├── deep_research_agent.md
│ ├── exa_research_testing_strategy.md
│ ├── PORTFOLIO_PERSONALIZATION_PLAN.md
│ ├── PORTFOLIO.md
│ ├── SETUP_SELF_CONTAINED.md
│ └── speed_testing_framework.md
├── examples
│ ├── complete_speed_validation.py
│ ├── deep_research_integration.py
│ ├── llm_optimization_example.py
│ ├── llm_speed_demo.py
│ ├── monitoring_example.py
│ ├── parallel_research_example.py
│ ├── speed_optimization_demo.py
│ └── timeout_fix_demonstration.py
├── LICENSE
├── Makefile
├── MANIFEST.in
├── maverick_mcp
│ ├── __init__.py
│ ├── agents
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── circuit_breaker.py
│ │ ├── deep_research.py
│ │ ├── market_analysis.py
│ │ ├── optimized_research.py
│ │ ├── supervisor.py
│ │ └── technical_analysis.py
│ ├── api
│ │ ├── __init__.py
│ │ ├── api_server.py
│ │ ├── connection_manager.py
│ │ ├── dependencies
│ │ │ ├── __init__.py
│ │ │ ├── stock_analysis.py
│ │ │ └── technical_analysis.py
│ │ ├── error_handling.py
│ │ ├── inspector_compatible_sse.py
│ │ ├── inspector_sse.py
│ │ ├── middleware
│ │ │ ├── error_handling.py
│ │ │ ├── mcp_logging.py
│ │ │ ├── rate_limiting_enhanced.py
│ │ │ └── security.py
│ │ ├── openapi_config.py
│ │ ├── routers
│ │ │ ├── __init__.py
│ │ │ ├── agents.py
│ │ │ ├── backtesting.py
│ │ │ ├── data_enhanced.py
│ │ │ ├── data.py
│ │ │ ├── health_enhanced.py
│ │ │ ├── health_tools.py
│ │ │ ├── health.py
│ │ │ ├── intelligent_backtesting.py
│ │ │ ├── introspection.py
│ │ │ ├── mcp_prompts.py
│ │ │ ├── monitoring.py
│ │ │ ├── news_sentiment_enhanced.py
│ │ │ ├── performance.py
│ │ │ ├── portfolio.py
│ │ │ ├── research.py
│ │ │ ├── screening_ddd.py
│ │ │ ├── screening_parallel.py
│ │ │ ├── screening.py
│ │ │ ├── technical_ddd.py
│ │ │ ├── technical_enhanced.py
│ │ │ ├── technical.py
│ │ │ └── tool_registry.py
│ │ ├── server.py
│ │ ├── services
│ │ │ ├── __init__.py
│ │ │ ├── base_service.py
│ │ │ ├── market_service.py
│ │ │ ├── portfolio_service.py
│ │ │ ├── prompt_service.py
│ │ │ └── resource_service.py
│ │ ├── simple_sse.py
│ │ └── utils
│ │ ├── __init__.py
│ │ ├── insomnia_export.py
│ │ └── postman_export.py
│ ├── application
│ │ ├── __init__.py
│ │ ├── commands
│ │ │ └── __init__.py
│ │ ├── dto
│ │ │ ├── __init__.py
│ │ │ └── technical_analysis_dto.py
│ │ ├── queries
│ │ │ ├── __init__.py
│ │ │ └── get_technical_analysis.py
│ │ └── screening
│ │ ├── __init__.py
│ │ ├── dtos.py
│ │ └── queries.py
│ ├── backtesting
│ │ ├── __init__.py
│ │ ├── ab_testing.py
│ │ ├── analysis.py
│ │ ├── batch_processing_stub.py
│ │ ├── batch_processing.py
│ │ ├── model_manager.py
│ │ ├── optimization.py
│ │ ├── persistence.py
│ │ ├── retraining_pipeline.py
│ │ ├── strategies
│ │ │ ├── __init__.py
│ │ │ ├── base.py
│ │ │ ├── ml
│ │ │ │ ├── __init__.py
│ │ │ │ ├── adaptive.py
│ │ │ │ ├── ensemble.py
│ │ │ │ ├── feature_engineering.py
│ │ │ │ └── regime_aware.py
│ │ │ ├── ml_strategies.py
│ │ │ ├── parser.py
│ │ │ └── templates.py
│ │ ├── strategy_executor.py
│ │ ├── vectorbt_engine.py
│ │ └── visualization.py
│ ├── config
│ │ ├── __init__.py
│ │ ├── constants.py
│ │ ├── database_self_contained.py
│ │ ├── database.py
│ │ ├── llm_optimization_config.py
│ │ ├── logging_settings.py
│ │ ├── plotly_config.py
│ │ ├── security_utils.py
│ │ ├── security.py
│ │ ├── settings.py
│ │ ├── technical_constants.py
│ │ ├── tool_estimation.py
│ │ └── validation.py
│ ├── core
│ │ ├── __init__.py
│ │ ├── technical_analysis.py
│ │ └── visualization.py
│ ├── data
│ │ ├── __init__.py
│ │ ├── cache_manager.py
│ │ ├── cache.py
│ │ ├── django_adapter.py
│ │ ├── health.py
│ │ ├── models.py
│ │ ├── performance.py
│ │ ├── session_management.py
│ │ └── validation.py
│ ├── database
│ │ ├── __init__.py
│ │ ├── base.py
│ │ └── optimization.py
│ ├── dependencies.py
│ ├── domain
│ │ ├── __init__.py
│ │ ├── entities
│ │ │ ├── __init__.py
│ │ │ └── stock_analysis.py
│ │ ├── events
│ │ │ └── __init__.py
│ │ ├── portfolio.py
│ │ ├── screening
│ │ │ ├── __init__.py
│ │ │ ├── entities.py
│ │ │ ├── services.py
│ │ │ └── value_objects.py
│ │ ├── services
│ │ │ ├── __init__.py
│ │ │ └── technical_analysis_service.py
│ │ ├── stock_analysis
│ │ │ ├── __init__.py
│ │ │ └── stock_analysis_service.py
│ │ └── value_objects
│ │ ├── __init__.py
│ │ └── technical_indicators.py
│ ├── exceptions.py
│ ├── infrastructure
│ │ ├── __init__.py
│ │ ├── cache
│ │ │ └── __init__.py
│ │ ├── caching
│ │ │ ├── __init__.py
│ │ │ └── cache_management_service.py
│ │ ├── connection_manager.py
│ │ ├── data_fetching
│ │ │ ├── __init__.py
│ │ │ └── stock_data_service.py
│ │ ├── health
│ │ │ ├── __init__.py
│ │ │ └── health_checker.py
│ │ ├── persistence
│ │ │ ├── __init__.py
│ │ │ └── stock_repository.py
│ │ ├── providers
│ │ │ └── __init__.py
│ │ ├── screening
│ │ │ ├── __init__.py
│ │ │ └── repositories.py
│ │ └── sse_optimizer.py
│ ├── langchain_tools
│ │ ├── __init__.py
│ │ ├── adapters.py
│ │ └── registry.py
│ ├── logging_config.py
│ ├── memory
│ │ ├── __init__.py
│ │ └── stores.py
│ ├── monitoring
│ │ ├── __init__.py
│ │ ├── health_check.py
│ │ ├── health_monitor.py
│ │ ├── integration_example.py
│ │ ├── metrics.py
│ │ ├── middleware.py
│ │ └── status_dashboard.py
│ ├── providers
│ │ ├── __init__.py
│ │ ├── dependencies.py
│ │ ├── factories
│ │ │ ├── __init__.py
│ │ │ ├── config_factory.py
│ │ │ └── provider_factory.py
│ │ ├── implementations
│ │ │ ├── __init__.py
│ │ │ ├── cache_adapter.py
│ │ │ ├── macro_data_adapter.py
│ │ │ ├── market_data_adapter.py
│ │ │ ├── persistence_adapter.py
│ │ │ └── stock_data_adapter.py
│ │ ├── interfaces
│ │ │ ├── __init__.py
│ │ │ ├── cache.py
│ │ │ ├── config.py
│ │ │ ├── macro_data.py
│ │ │ ├── market_data.py
│ │ │ ├── persistence.py
│ │ │ └── stock_data.py
│ │ ├── llm_factory.py
│ │ ├── macro_data.py
│ │ ├── market_data.py
│ │ ├── mocks
│ │ │ ├── __init__.py
│ │ │ ├── mock_cache.py
│ │ │ ├── mock_config.py
│ │ │ ├── mock_macro_data.py
│ │ │ ├── mock_market_data.py
│ │ │ ├── mock_persistence.py
│ │ │ └── mock_stock_data.py
│ │ ├── openrouter_provider.py
│ │ ├── optimized_screening.py
│ │ ├── optimized_stock_data.py
│ │ └── stock_data.py
│ ├── README.md
│ ├── tests
│ │ ├── __init__.py
│ │ ├── README_INMEMORY_TESTS.md
│ │ ├── test_cache_debug.py
│ │ ├── test_fixes_validation.py
│ │ ├── test_in_memory_routers.py
│ │ ├── test_in_memory_server.py
│ │ ├── test_macro_data_provider.py
│ │ ├── test_mailgun_email.py
│ │ ├── test_market_calendar_caching.py
│ │ ├── test_mcp_tool_fixes_pytest.py
│ │ ├── test_mcp_tool_fixes.py
│ │ ├── test_mcp_tools.py
│ │ ├── test_models_functional.py
│ │ ├── test_server.py
│ │ ├── test_stock_data_enhanced.py
│ │ ├── test_stock_data_provider.py
│ │ └── test_technical_analysis.py
│ ├── tools
│ │ ├── __init__.py
│ │ ├── performance_monitoring.py
│ │ ├── portfolio_manager.py
│ │ ├── risk_management.py
│ │ └── sentiment_analysis.py
│ ├── utils
│ │ ├── __init__.py
│ │ ├── agent_errors.py
│ │ ├── batch_processing.py
│ │ ├── cache_warmer.py
│ │ ├── circuit_breaker_decorators.py
│ │ ├── circuit_breaker_services.py
│ │ ├── circuit_breaker.py
│ │ ├── data_chunking.py
│ │ ├── database_monitoring.py
│ │ ├── debug_utils.py
│ │ ├── fallback_strategies.py
│ │ ├── llm_optimization.py
│ │ ├── logging_example.py
│ │ ├── logging_init.py
│ │ ├── logging.py
│ │ ├── mcp_logging.py
│ │ ├── memory_profiler.py
│ │ ├── monitoring_middleware.py
│ │ ├── monitoring.py
│ │ ├── orchestration_logging.py
│ │ ├── parallel_research.py
│ │ ├── parallel_screening.py
│ │ ├── quick_cache.py
│ │ ├── resource_manager.py
│ │ ├── shutdown.py
│ │ ├── stock_helpers.py
│ │ ├── structured_logger.py
│ │ ├── tool_monitoring.py
│ │ ├── tracing.py
│ │ └── yfinance_pool.py
│ ├── validation
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── data.py
│ │ ├── middleware.py
│ │ ├── portfolio.py
│ │ ├── responses.py
│ │ ├── screening.py
│ │ └── technical.py
│ └── workflows
│ ├── __init__.py
│ ├── agents
│ │ ├── __init__.py
│ │ ├── market_analyzer.py
│ │ ├── optimizer_agent.py
│ │ ├── strategy_selector.py
│ │ └── validator_agent.py
│ ├── backtesting_workflow.py
│ └── state.py
├── PLANS.md
├── pyproject.toml
├── pyrightconfig.json
├── README.md
├── scripts
│ ├── dev.sh
│ ├── INSTALLATION_GUIDE.md
│ ├── load_example.py
│ ├── load_market_data.py
│ ├── load_tiingo_data.py
│ ├── migrate_db.py
│ ├── README_TIINGO_LOADER.md
│ ├── requirements_tiingo.txt
│ ├── run_stock_screening.py
│ ├── run-migrations.sh
│ ├── seed_db.py
│ ├── seed_sp500.py
│ ├── setup_database.sh
│ ├── setup_self_contained.py
│ ├── setup_sp500_database.sh
│ ├── test_seeded_data.py
│ ├── test_tiingo_loader.py
│ ├── tiingo_config.py
│ └── validate_setup.py
├── SECURITY.md
├── server.json
├── setup.py
├── tests
│ ├── __init__.py
│ ├── conftest.py
│ ├── core
│ │ └── test_technical_analysis.py
│ ├── data
│ │ └── test_portfolio_models.py
│ ├── domain
│ │ ├── conftest.py
│ │ ├── test_portfolio_entities.py
│ │ └── test_technical_analysis_service.py
│ ├── fixtures
│ │ └── orchestration_fixtures.py
│ ├── integration
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── README.md
│ │ ├── run_integration_tests.sh
│ │ ├── test_api_technical.py
│ │ ├── test_chaos_engineering.py
│ │ ├── test_config_management.py
│ │ ├── test_full_backtest_workflow_advanced.py
│ │ ├── test_full_backtest_workflow.py
│ │ ├── test_high_volume.py
│ │ ├── test_mcp_tools.py
│ │ ├── test_orchestration_complete.py
│ │ ├── test_portfolio_persistence.py
│ │ ├── test_redis_cache.py
│ │ ├── test_security_integration.py.disabled
│ │ └── vcr_setup.py
│ ├── performance
│ │ ├── __init__.py
│ │ ├── test_benchmarks.py
│ │ ├── test_load.py
│ │ ├── test_profiling.py
│ │ └── test_stress.py
│ ├── providers
│ │ └── test_stock_data_simple.py
│ ├── README.md
│ ├── test_agents_router_mcp.py
│ ├── test_backtest_persistence.py
│ ├── test_cache_management_service.py
│ ├── test_cache_serialization.py
│ ├── test_circuit_breaker.py
│ ├── test_database_pool_config_simple.py
│ ├── test_database_pool_config.py
│ ├── test_deep_research_functional.py
│ ├── test_deep_research_integration.py
│ ├── test_deep_research_parallel_execution.py
│ ├── test_error_handling.py
│ ├── test_event_loop_integrity.py
│ ├── test_exa_research_integration.py
│ ├── test_exception_hierarchy.py
│ ├── test_financial_search.py
│ ├── test_graceful_shutdown.py
│ ├── test_integration_simple.py
│ ├── test_langgraph_workflow.py
│ ├── test_market_data_async.py
│ ├── test_market_data_simple.py
│ ├── test_mcp_orchestration_functional.py
│ ├── test_ml_strategies.py
│ ├── test_optimized_research_agent.py
│ ├── test_orchestration_integration.py
│ ├── test_orchestration_logging.py
│ ├── test_orchestration_tools_simple.py
│ ├── test_parallel_research_integration.py
│ ├── test_parallel_research_orchestrator.py
│ ├── test_parallel_research_performance.py
│ ├── test_performance_optimizations.py
│ ├── test_production_validation.py
│ ├── test_provider_architecture.py
│ ├── test_rate_limiting_enhanced.py
│ ├── test_runner_validation.py
│ ├── test_security_comprehensive.py.disabled
│ ├── test_security_cors.py
│ ├── test_security_enhancements.py.disabled
│ ├── test_security_headers.py
│ ├── test_security_penetration.py
│ ├── test_session_management.py
│ ├── test_speed_optimization_validation.py
│ ├── test_stock_analysis_dependencies.py
│ ├── test_stock_analysis_service.py
│ ├── test_stock_data_fetching_service.py
│ ├── test_supervisor_agent.py
│ ├── test_supervisor_functional.py
│ ├── test_tool_estimation_config.py
│ ├── test_visualization.py
│ └── utils
│ ├── test_agent_errors.py
│ ├── test_logging.py
│ ├── test_parallel_screening.py
│ └── test_quick_cache.py
├── tools
│ ├── check_orchestration_config.py
│ ├── experiments
│ │ ├── validation_examples.py
│ │ └── validation_fixed.py
│ ├── fast_dev.sh
│ ├── hot_reload.py
│ ├── quick_test.py
│ └── templates
│ ├── new_router_template.py
│ ├── new_tool_template.py
│ ├── screening_strategy_template.py
│ └── test_template.py
└── uv.lock
```
# Files
--------------------------------------------------------------------------------
/maverick_mcp/api/routers/health_tools.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | MCP tools for health monitoring and system status.
3 |
4 | These tools expose health monitoring functionality through the MCP interface,
5 | allowing Claude to check system health, monitor component status, and get
6 | real-time metrics about the backtesting system.
7 | """
8 |
9 | import logging
10 | from datetime import UTC, datetime
11 | from typing import Any
12 |
13 | from fastmcp import FastMCP
14 |
15 | from maverick_mcp.config.settings import get_settings
16 |
17 | logger = logging.getLogger(__name__)
18 | settings = get_settings()
19 |
20 |
21 | def register_health_tools(mcp: FastMCP):
22 | """Register all health monitoring tools with the MCP server."""
23 |
24 | @mcp.tool()
25 | async def get_system_health() -> dict[str, Any]:
26 | """
27 | Get comprehensive system health status.
28 |
29 | Returns detailed information about all system components including:
30 | - Overall health status
31 | - Component-by-component status
32 | - Resource utilization
33 | - Circuit breaker states
34 | - Performance metrics
35 |
36 | Returns:
37 | Dictionary containing complete system health information
38 | """
39 | try:
40 | from maverick_mcp.api.routers.health_enhanced import (
41 | _get_detailed_health_status,
42 | )
43 |
44 | health_status = await _get_detailed_health_status()
45 | return {
46 | "status": "success",
47 | "data": health_status,
48 | "timestamp": datetime.now(UTC).isoformat(),
49 | }
50 |
51 | except Exception as e:
52 | logger.error(f"Failed to get system health: {e}")
53 | return {
54 | "status": "error",
55 | "error": str(e),
56 | "timestamp": datetime.now(UTC).isoformat(),
57 | }
58 |
59 | @mcp.tool()
60 | async def get_component_status(component_name: str = None) -> dict[str, Any]:
61 | """
62 | Get status of a specific component or all components.
63 |
64 | Args:
65 | component_name: Name of the component to check (optional).
66 | If not provided, returns status of all components.
67 |
68 | Returns:
69 | Dictionary containing component status information
70 | """
71 | try:
72 | from maverick_mcp.api.routers.health_enhanced import (
73 | _get_detailed_health_status,
74 | )
75 |
76 | health_status = await _get_detailed_health_status()
77 | components = health_status.get("components", {})
78 |
79 | if component_name:
80 | if component_name in components:
81 | return {
82 | "status": "success",
83 | "component": component_name,
84 | "data": components[component_name].__dict__,
85 | "timestamp": datetime.now(UTC).isoformat(),
86 | }
87 | else:
88 | return {
89 | "status": "error",
90 | "error": f"Component '{component_name}' not found",
91 | "available_components": list(components.keys()),
92 | "timestamp": datetime.now(UTC).isoformat(),
93 | }
94 | else:
95 | return {
96 | "status": "success",
97 | "data": {name: comp.__dict__ for name, comp in components.items()},
98 | "total_components": len(components),
99 | "timestamp": datetime.now(UTC).isoformat(),
100 | }
101 |
102 | except Exception as e:
103 | logger.error(f"Failed to get component status: {e}")
104 | return {
105 | "status": "error",
106 | "error": str(e),
107 | "timestamp": datetime.now(UTC).isoformat(),
108 | }
109 |
110 | @mcp.tool()
111 | async def get_circuit_breaker_status() -> dict[str, Any]:
112 | """
113 | Get status of all circuit breakers.
114 |
115 | Returns information about circuit breaker states, failure counts,
116 | and performance metrics for all external API connections.
117 |
118 | Returns:
119 | Dictionary containing circuit breaker status information
120 | """
121 | try:
122 | from maverick_mcp.utils.circuit_breaker import (
123 | get_all_circuit_breaker_status,
124 | )
125 |
126 | cb_status = get_all_circuit_breaker_status()
127 |
128 | return {
129 | "status": "success",
130 | "data": cb_status,
131 | "summary": {
132 | "total_breakers": len(cb_status),
133 | "states": {
134 | "closed": sum(
135 | 1
136 | for cb in cb_status.values()
137 | if cb.get("state") == "closed"
138 | ),
139 | "open": sum(
140 | 1 for cb in cb_status.values() if cb.get("state") == "open"
141 | ),
142 | "half_open": sum(
143 | 1
144 | for cb in cb_status.values()
145 | if cb.get("state") == "half_open"
146 | ),
147 | },
148 | },
149 | "timestamp": datetime.now(UTC).isoformat(),
150 | }
151 |
152 | except Exception as e:
153 | logger.error(f"Failed to get circuit breaker status: {e}")
154 | return {
155 | "status": "error",
156 | "error": str(e),
157 | "timestamp": datetime.now(UTC).isoformat(),
158 | }
159 |
160 | @mcp.tool()
161 | async def get_resource_usage() -> dict[str, Any]:
162 | """
163 | Get current system resource usage.
164 |
165 | Returns information about CPU, memory, disk usage, and other
166 | system resources being consumed by the backtesting system.
167 |
168 | Returns:
169 | Dictionary containing resource usage information
170 | """
171 | try:
172 | from maverick_mcp.api.routers.health_enhanced import _get_resource_usage
173 |
174 | resource_usage = _get_resource_usage()
175 |
176 | return {
177 | "status": "success",
178 | "data": resource_usage.__dict__,
179 | "alerts": {
180 | "high_cpu": resource_usage.cpu_percent > 80,
181 | "high_memory": resource_usage.memory_percent > 85,
182 | "high_disk": resource_usage.disk_percent > 90,
183 | },
184 | "timestamp": datetime.now(UTC).isoformat(),
185 | }
186 |
187 | except Exception as e:
188 | logger.error(f"Failed to get resource usage: {e}")
189 | return {
190 | "status": "error",
191 | "error": str(e),
192 | "timestamp": datetime.now(UTC).isoformat(),
193 | }
194 |
195 | @mcp.tool()
196 | async def get_status_dashboard() -> dict[str, Any]:
197 | """
198 | Get comprehensive status dashboard data.
199 |
200 | Returns aggregated health status, performance metrics, alerts,
201 | and historical trends for the entire backtesting system.
202 |
203 | Returns:
204 | Dictionary containing complete dashboard information
205 | """
206 | try:
207 | from maverick_mcp.monitoring.status_dashboard import get_dashboard_data
208 |
209 | dashboard_data = await get_dashboard_data()
210 |
211 | return {
212 | "status": "success",
213 | "data": dashboard_data,
214 | "timestamp": datetime.now(UTC).isoformat(),
215 | }
216 |
217 | except Exception as e:
218 | logger.error(f"Failed to get status dashboard: {e}")
219 | return {
220 | "status": "error",
221 | "error": str(e),
222 | "timestamp": datetime.now(UTC).isoformat(),
223 | }
224 |
225 | @mcp.tool()
226 | async def reset_circuit_breaker(breaker_name: str) -> dict[str, Any]:
227 | """
228 | Reset a specific circuit breaker.
229 |
230 | Args:
231 | breaker_name: Name of the circuit breaker to reset
232 |
233 | Returns:
234 | Dictionary containing operation result
235 | """
236 | try:
237 | from maverick_mcp.utils.circuit_breaker import get_circuit_breaker_manager
238 |
239 | manager = get_circuit_breaker_manager()
240 | success = manager.reset_breaker(breaker_name)
241 |
242 | if success:
243 | return {
244 | "status": "success",
245 | "message": f"Circuit breaker '{breaker_name}' reset successfully",
246 | "breaker_name": breaker_name,
247 | "timestamp": datetime.now(UTC).isoformat(),
248 | }
249 | else:
250 | return {
251 | "status": "error",
252 | "error": f"Circuit breaker '{breaker_name}' not found or could not be reset",
253 | "breaker_name": breaker_name,
254 | "timestamp": datetime.now(UTC).isoformat(),
255 | }
256 |
257 | except Exception as e:
258 | logger.error(f"Failed to reset circuit breaker {breaker_name}: {e}")
259 | return {
260 | "status": "error",
261 | "error": str(e),
262 | "breaker_name": breaker_name,
263 | "timestamp": datetime.now(UTC).isoformat(),
264 | }
265 |
266 | @mcp.tool()
267 | async def get_health_history() -> dict[str, Any]:
268 | """
269 | Get historical health data for trend analysis.
270 |
271 | Returns recent health check history including component status
272 | changes, resource usage trends, and system performance over time.
273 |
274 | Returns:
275 | Dictionary containing historical health information
276 | """
277 | try:
278 | from maverick_mcp.monitoring.health_monitor import get_health_monitor
279 |
280 | monitor = get_health_monitor()
281 | monitoring_status = monitor.get_monitoring_status()
282 |
283 | # Get historical data from dashboard
284 | from maverick_mcp.monitoring.status_dashboard import get_status_dashboard
285 |
286 | dashboard = get_status_dashboard()
287 | dashboard_data = await dashboard.get_dashboard_data()
288 | historical_data = dashboard_data.get("historical", {})
289 |
290 | return {
291 | "status": "success",
292 | "data": {
293 | "monitoring_status": monitoring_status,
294 | "historical_data": historical_data,
295 | "trends": {
296 | "data_points": len(historical_data.get("data", [])),
297 | "timespan_hours": historical_data.get("summary", {}).get(
298 | "timespan_hours", 0
299 | ),
300 | },
301 | },
302 | "timestamp": datetime.now(UTC).isoformat(),
303 | }
304 |
305 | except Exception as e:
306 | logger.error(f"Failed to get health history: {e}")
307 | return {
308 | "status": "error",
309 | "error": str(e),
310 | "timestamp": datetime.now(UTC).isoformat(),
311 | }
312 |
313 | @mcp.tool()
314 | async def run_health_diagnostics() -> dict[str, Any]:
315 | """
316 | Run comprehensive health diagnostics.
317 |
318 | Performs a complete system health check including all components,
319 | circuit breakers, resource usage, and generates a diagnostic report
320 | with recommendations.
321 |
322 | Returns:
323 | Dictionary containing diagnostic results and recommendations
324 | """
325 | try:
326 | # Get all health information
327 | from maverick_mcp.api.routers.health_enhanced import (
328 | _get_detailed_health_status,
329 | )
330 | from maverick_mcp.monitoring.health_monitor import get_monitoring_status
331 | from maverick_mcp.utils.circuit_breaker import (
332 | get_all_circuit_breaker_status,
333 | )
334 |
335 | health_status = await _get_detailed_health_status()
336 | cb_status = get_all_circuit_breaker_status()
337 | monitoring_status = get_monitoring_status()
338 |
339 | # Generate recommendations
340 | recommendations = []
341 |
342 | # Check component health
343 | components = health_status.get("components", {})
344 | unhealthy_components = [
345 | name for name, comp in components.items() if comp.status == "unhealthy"
346 | ]
347 | if unhealthy_components:
348 | recommendations.append(
349 | {
350 | "type": "component_health",
351 | "severity": "critical",
352 | "message": f"Unhealthy components detected: {', '.join(unhealthy_components)}",
353 | "action": "Check component logs and dependencies",
354 | }
355 | )
356 |
357 | # Check circuit breakers
358 | open_breakers = [
359 | name for name, cb in cb_status.items() if cb.get("state") == "open"
360 | ]
361 | if open_breakers:
362 | recommendations.append(
363 | {
364 | "type": "circuit_breaker",
365 | "severity": "warning",
366 | "message": f"Open circuit breakers: {', '.join(open_breakers)}",
367 | "action": "Check external service availability and consider resetting breakers",
368 | }
369 | )
370 |
371 | # Check resource usage
372 | resource_usage = health_status.get("resource_usage", {})
373 | if resource_usage.get("memory_percent", 0) > 85:
374 | recommendations.append(
375 | {
376 | "type": "resource_usage",
377 | "severity": "warning",
378 | "message": f"High memory usage: {resource_usage.get('memory_percent')}%",
379 | "action": "Monitor memory usage trends and consider scaling",
380 | }
381 | )
382 |
383 | if resource_usage.get("cpu_percent", 0) > 80:
384 | recommendations.append(
385 | {
386 | "type": "resource_usage",
387 | "severity": "warning",
388 | "message": f"High CPU usage: {resource_usage.get('cpu_percent')}%",
389 | "action": "Check for high-load operations and optimize if needed",
390 | }
391 | )
392 |
393 | # Generate overall assessment
394 | overall_health_score = 100
395 | if unhealthy_components:
396 | overall_health_score -= len(unhealthy_components) * 20
397 | if open_breakers:
398 | overall_health_score -= len(open_breakers) * 10
399 | if resource_usage.get("memory_percent", 0) > 85:
400 | overall_health_score -= 15
401 | if resource_usage.get("cpu_percent", 0) > 80:
402 | overall_health_score -= 10
403 |
404 | overall_health_score = max(0, overall_health_score)
405 |
406 | return {
407 | "status": "success",
408 | "data": {
409 | "overall_health_score": overall_health_score,
410 | "system_status": health_status.get("status", "unknown"),
411 | "component_summary": {
412 | "total": len(components),
413 | "healthy": sum(
414 | 1 for c in components.values() if c.status == "healthy"
415 | ),
416 | "degraded": sum(
417 | 1 for c in components.values() if c.status == "degraded"
418 | ),
419 | "unhealthy": sum(
420 | 1 for c in components.values() if c.status == "unhealthy"
421 | ),
422 | },
423 | "circuit_breaker_summary": {
424 | "total": len(cb_status),
425 | "closed": sum(
426 | 1
427 | for cb in cb_status.values()
428 | if cb.get("state") == "closed"
429 | ),
430 | "open": len(open_breakers),
431 | "half_open": sum(
432 | 1
433 | for cb in cb_status.values()
434 | if cb.get("state") == "half_open"
435 | ),
436 | },
437 | "resource_summary": resource_usage,
438 | "monitoring_summary": monitoring_status,
439 | "recommendations": recommendations,
440 | },
441 | "timestamp": datetime.now(UTC).isoformat(),
442 | }
443 |
444 | except Exception as e:
445 | logger.error(f"Failed to run health diagnostics: {e}")
446 | return {
447 | "status": "error",
448 | "error": str(e),
449 | "timestamp": datetime.now(UTC).isoformat(),
450 | }
451 |
452 | logger.info("Health monitoring tools registered successfully")
453 |
```
--------------------------------------------------------------------------------
/maverick_mcp/tools/performance_monitoring.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Performance monitoring tools for Maverick-MCP.
3 |
4 | This module provides MCP tools for monitoring and analyzing system performance,
5 | including Redis connection health, cache hit rates, query performance, and
6 | database index usage.
7 | """
8 |
9 | import logging
10 | import time
11 | from datetime import datetime
12 | from typing import Any
13 |
14 | from sqlalchemy import text
15 |
16 | from maverick_mcp.data.performance import (
17 | query_optimizer,
18 | redis_manager,
19 | request_cache,
20 | )
21 | from maverick_mcp.data.session_management import (
22 | get_async_connection_pool_status,
23 | get_async_db_session,
24 | )
25 | from maverick_mcp.providers.optimized_stock_data import optimized_stock_provider
26 |
27 | logger = logging.getLogger(__name__)
28 |
29 |
30 | async def get_redis_connection_health() -> dict[str, Any]:
31 | """
32 | Get comprehensive Redis connection health metrics.
33 |
34 | Returns:
35 | Dictionary with Redis health information
36 | """
37 | try:
38 | metrics = redis_manager.get_metrics()
39 |
40 | # Test basic Redis operations
41 | test_key = f"health_check_{int(time.time())}"
42 | test_value = "test_value"
43 |
44 | client = await redis_manager.get_client()
45 | if client:
46 | # Test basic operations
47 | start_time = time.time()
48 | await client.set(test_key, test_value, ex=60) # 1 minute expiry
49 | get_result = await client.get(test_key)
50 | await client.delete(test_key)
51 | operation_time = time.time() - start_time
52 |
53 | redis_health = {
54 | "status": "healthy",
55 | "basic_operations_working": get_result == test_value,
56 | "operation_latency_ms": round(operation_time * 1000, 2),
57 | }
58 | else:
59 | redis_health = {
60 | "status": "unhealthy",
61 | "basic_operations_working": False,
62 | "operation_latency_ms": None,
63 | }
64 |
65 | return {
66 | "redis_health": redis_health,
67 | "connection_metrics": metrics,
68 | "timestamp": datetime.now().isoformat(),
69 | }
70 |
71 | except Exception as e:
72 | logger.error(f"Error checking Redis health: {e}")
73 | return {
74 | "redis_health": {
75 | "status": "error",
76 | "error": str(e),
77 | "basic_operations_working": False,
78 | "operation_latency_ms": None,
79 | },
80 | "connection_metrics": {},
81 | "timestamp": datetime.now().isoformat(),
82 | }
83 |
84 |
85 | async def get_cache_performance_metrics() -> dict[str, Any]:
86 | """
87 | Get comprehensive cache performance metrics.
88 |
89 | Returns:
90 | Dictionary with cache performance information
91 | """
92 | try:
93 | # Get basic cache metrics
94 | cache_metrics = request_cache.get_metrics()
95 |
96 | # Test cache performance
97 | test_key = f"cache_perf_test_{int(time.time())}"
98 | test_data = {"test": "data", "timestamp": time.time()}
99 |
100 | # Test cache operations
101 | start_time = time.time()
102 | set_success = await request_cache.set(test_key, test_data, ttl=60)
103 | set_time = time.time() - start_time
104 |
105 | start_time = time.time()
106 | retrieved_data = await request_cache.get(test_key)
107 | get_time = time.time() - start_time
108 |
109 | # Cleanup
110 | await request_cache.delete(test_key)
111 |
112 | performance_test = {
113 | "set_operation_ms": round(set_time * 1000, 2),
114 | "get_operation_ms": round(get_time * 1000, 2),
115 | "set_success": set_success,
116 | "get_success": retrieved_data is not None,
117 | "data_integrity": retrieved_data == test_data if retrieved_data else False,
118 | }
119 |
120 | # Get Redis-specific metrics if available
121 | redis_metrics = redis_manager.get_metrics()
122 |
123 | return {
124 | "cache_performance": cache_metrics,
125 | "performance_test": performance_test,
126 | "redis_metrics": redis_metrics,
127 | "timestamp": datetime.now().isoformat(),
128 | }
129 |
130 | except Exception as e:
131 | logger.error(f"Error getting cache performance metrics: {e}")
132 | return {
133 | "error": str(e),
134 | "timestamp": datetime.now().isoformat(),
135 | }
136 |
137 |
138 | async def get_query_performance_metrics() -> dict[str, Any]:
139 | """
140 | Get database query performance metrics.
141 |
142 | Returns:
143 | Dictionary with query performance information
144 | """
145 | try:
146 | # Get query optimizer stats
147 | query_stats = query_optimizer.get_query_stats()
148 |
149 | # Get database connection pool stats
150 | try:
151 | async with get_async_db_session() as session:
152 | pool_status = await get_async_connection_pool_status()
153 |
154 | start_time = time.time()
155 | result = await session.execute(text("SELECT 1 as test"))
156 | result.fetchone()
157 | db_latency = time.time() - start_time
158 |
159 | db_health = {
160 | "status": "healthy",
161 | "latency_ms": round(db_latency * 1000, 2),
162 | "pool_status": pool_status,
163 | }
164 | except Exception as e:
165 | db_health = {
166 | "status": "unhealthy",
167 | "error": str(e),
168 | "latency_ms": None,
169 | "pool_status": {},
170 | }
171 |
172 | return {
173 | "query_performance": query_stats,
174 | "database_health": db_health,
175 | "timestamp": datetime.now().isoformat(),
176 | }
177 |
178 | except Exception as e:
179 | logger.error(f"Error getting query performance metrics: {e}")
180 | return {
181 | "error": str(e),
182 | "timestamp": datetime.now().isoformat(),
183 | }
184 |
185 |
186 | async def analyze_database_indexes() -> dict[str, Any]:
187 | """
188 | Analyze database index usage and provide recommendations.
189 |
190 | Returns:
191 | Dictionary with index analysis and recommendations
192 | """
193 | try:
194 | async with get_async_db_session() as session:
195 | recommendations = await query_optimizer.analyze_missing_indexes(session)
196 |
197 | # Get index usage statistics
198 | index_usage_query = text(
199 | """
200 | SELECT
201 | schemaname,
202 | tablename,
203 | indexname,
204 | idx_scan,
205 | idx_tup_read,
206 | idx_tup_fetch
207 | FROM pg_stat_user_indexes
208 | WHERE schemaname = 'public'
209 | AND tablename LIKE 'stocks_%'
210 | ORDER BY idx_scan DESC
211 | """
212 | )
213 |
214 | result = await session.execute(index_usage_query)
215 | index_usage = [dict(row._mapping) for row in result.fetchall()] # type: ignore[attr-defined]
216 |
217 | # Get table scan statistics
218 | table_scan_query = text(
219 | """
220 | SELECT
221 | schemaname,
222 | tablename,
223 | seq_scan,
224 | seq_tup_read,
225 | idx_scan,
226 | idx_tup_fetch,
227 | CASE
228 | WHEN seq_scan + idx_scan = 0 THEN 0
229 | ELSE ROUND(100.0 * idx_scan / (seq_scan + idx_scan), 2)
230 | END as index_usage_percent
231 | FROM pg_stat_user_tables
232 | WHERE schemaname = 'public'
233 | AND tablename LIKE 'stocks_%'
234 | ORDER BY seq_tup_read DESC
235 | """
236 | )
237 |
238 | result = await session.execute(table_scan_query)
239 | table_stats = [dict(row._mapping) for row in result.fetchall()] # type: ignore[attr-defined]
240 |
241 | # Identify tables with poor index usage
242 | poor_index_usage = [
243 | table
244 | for table in table_stats
245 | if table["index_usage_percent"] < 80 and table["seq_scan"] > 100
246 | ]
247 |
248 | return {
249 | "index_recommendations": recommendations,
250 | "index_usage_stats": index_usage,
251 | "table_scan_stats": table_stats,
252 | "poor_index_usage": poor_index_usage,
253 | "analysis_timestamp": datetime.now().isoformat(),
254 | }
255 |
256 | except Exception as e:
257 | logger.error(f"Error analyzing database indexes: {e}")
258 | return {
259 | "error": str(e),
260 | "timestamp": datetime.now().isoformat(),
261 | }
262 |
263 |
264 | async def get_comprehensive_performance_report() -> dict[str, Any]:
265 | """
266 | Get a comprehensive performance report combining all metrics.
267 |
268 | Returns:
269 | Dictionary with complete performance analysis
270 | """
271 | try:
272 | # Gather all performance metrics
273 | redis_health = await get_redis_connection_health()
274 | cache_metrics = await get_cache_performance_metrics()
275 | query_metrics = await get_query_performance_metrics()
276 | index_analysis = await analyze_database_indexes()
277 | provider_metrics = await optimized_stock_provider.get_performance_metrics()
278 |
279 | # Calculate overall health scores
280 | redis_score = 100 if redis_health["redis_health"]["status"] == "healthy" else 0
281 |
282 | cache_hit_rate = cache_metrics.get("cache_performance", {}).get("hit_rate", 0)
283 | cache_score = cache_hit_rate * 100
284 |
285 | # Database performance score based on average query time
286 | query_stats = query_metrics.get("query_performance", {}).get("query_stats", {})
287 | avg_query_times = [stats.get("avg_time", 0) for stats in query_stats.values()]
288 | avg_query_time = (
289 | sum(avg_query_times) / len(avg_query_times) if avg_query_times else 0
290 | )
291 | db_score = max(0, 100 - (avg_query_time * 100)) # Penalty for slow queries
292 |
293 | overall_score = (redis_score + cache_score + db_score) / 3
294 |
295 | # Performance recommendations
296 | recommendations = []
297 |
298 | if redis_score < 100:
299 | recommendations.append(
300 | "Redis connection issues detected. Check Redis server status."
301 | )
302 |
303 | if cache_hit_rate < 0.8:
304 | recommendations.append(
305 | f"Cache hit rate is {cache_hit_rate:.1%}. Consider increasing TTL values or cache size."
306 | )
307 |
308 | if avg_query_time > 0.5:
309 | recommendations.append(
310 | f"Average query time is {avg_query_time:.2f}s. Consider adding database indexes."
311 | )
312 |
313 | poor_index_tables = index_analysis.get("poor_index_usage", [])
314 | if poor_index_tables:
315 | table_names = [table["tablename"] for table in poor_index_tables]
316 | recommendations.append(
317 | f"Poor index usage on tables: {', '.join(table_names)}"
318 | )
319 |
320 | if not recommendations:
321 | recommendations.append("System performance is optimal.")
322 |
323 | return {
324 | "overall_health_score": round(overall_score, 1),
325 | "component_scores": {
326 | "redis": redis_score,
327 | "cache": round(cache_score, 1),
328 | "database": round(db_score, 1),
329 | },
330 | "recommendations": recommendations,
331 | "detailed_metrics": {
332 | "redis_health": redis_health,
333 | "cache_performance": cache_metrics,
334 | "query_performance": query_metrics,
335 | "index_analysis": index_analysis,
336 | "provider_metrics": provider_metrics,
337 | },
338 | "report_timestamp": datetime.now().isoformat(),
339 | }
340 |
341 | except Exception as e:
342 | logger.error(f"Error generating comprehensive performance report: {e}")
343 | return {
344 | "error": str(e),
345 | "timestamp": datetime.now().isoformat(),
346 | }
347 |
348 |
349 | async def optimize_cache_settings() -> dict[str, Any]:
350 | """
351 | Analyze current cache usage and suggest optimal settings.
352 |
353 | Returns:
354 | Dictionary with cache optimization recommendations
355 | """
356 | try:
357 | # Get current cache metrics
358 | cache_metrics = request_cache.get_metrics()
359 |
360 | # Analyze cache performance patterns
361 | hit_rate = cache_metrics.get("hit_rate", 0)
362 | total_requests = cache_metrics.get("total_requests", 0)
363 |
364 | # Get Redis memory usage if available
365 | client = await redis_manager.get_client()
366 | redis_info = {}
367 | if client:
368 | try:
369 | redis_info = await client.info("memory")
370 | except Exception as e:
371 | logger.warning(f"Could not get Redis memory info: {e}")
372 |
373 | # Generate recommendations
374 | recommendations = []
375 | optimal_settings = {}
376 |
377 | if hit_rate < 0.7:
378 | recommendations.append("Increase cache TTL values to improve hit rate")
379 | optimal_settings["stock_data_ttl"] = 7200 # 2 hours instead of 1
380 | optimal_settings["screening_ttl"] = 14400 # 4 hours instead of 2
381 | elif hit_rate > 0.95:
382 | recommendations.append(
383 | "Consider reducing TTL values to ensure data freshness"
384 | )
385 | optimal_settings["stock_data_ttl"] = 1800 # 30 minutes
386 | optimal_settings["screening_ttl"] = 3600 # 1 hour
387 |
388 | if total_requests > 10000:
389 | recommendations.append(
390 | "High cache usage detected. Consider increasing Redis memory allocation"
391 | )
392 |
393 | # Memory usage recommendations
394 | if redis_info.get("used_memory"):
395 | used_memory_mb = int(redis_info["used_memory"]) / (1024 * 1024)
396 | if used_memory_mb > 100:
397 | recommendations.append(
398 | f"Redis using {used_memory_mb:.1f}MB. Monitor memory usage."
399 | )
400 |
401 | return {
402 | "current_performance": cache_metrics,
403 | "redis_memory_info": redis_info,
404 | "recommendations": recommendations,
405 | "optimal_settings": optimal_settings,
406 | "analysis_timestamp": datetime.now().isoformat(),
407 | }
408 |
409 | except Exception as e:
410 | logger.error(f"Error optimizing cache settings: {e}")
411 | return {
412 | "error": str(e),
413 | "timestamp": datetime.now().isoformat(),
414 | }
415 |
416 |
417 | async def clear_performance_caches(
418 | cache_types: list[str] | None = None,
419 | ) -> dict[str, Any]:
420 | """
421 | Clear specific performance caches for testing or maintenance.
422 |
423 | Args:
424 | cache_types: List of cache types to clear (stock_data, screening, market_data, all)
425 |
426 | Returns:
427 | Dictionary with cache clearing results
428 | """
429 | if cache_types is None:
430 | cache_types = ["all"]
431 |
432 | results = {}
433 | total_cleared = 0
434 |
435 | try:
436 | for cache_type in cache_types:
437 | if cache_type == "all":
438 | # Clear all caches
439 | cleared = await request_cache.delete_pattern("cache:*")
440 | results["all_caches"] = cleared
441 | total_cleared += cleared
442 |
443 | elif cache_type == "stock_data":
444 | # Clear stock data caches
445 | patterns = [
446 | "cache:*get_stock_basic_info*",
447 | "cache:*get_stock_price_data*",
448 | "cache:*bulk_get_stock_data*",
449 | ]
450 | cleared = 0
451 | for pattern in patterns:
452 | cleared += await request_cache.delete_pattern(pattern)
453 | results["stock_data"] = cleared
454 | total_cleared += cleared
455 |
456 | elif cache_type == "screening":
457 | # Clear screening caches
458 | patterns = [
459 | "cache:*get_maverick_recommendations*",
460 | "cache:*get_trending_recommendations*",
461 | ]
462 | cleared = 0
463 | for pattern in patterns:
464 | cleared += await request_cache.delete_pattern(pattern)
465 | results["screening"] = cleared
466 | total_cleared += cleared
467 |
468 | elif cache_type == "market_data":
469 | # Clear market data caches
470 | patterns = [
471 | "cache:*get_high_volume_stocks*",
472 | "cache:*market_data*",
473 | ]
474 | cleared = 0
475 | for pattern in patterns:
476 | cleared += await request_cache.delete_pattern(pattern)
477 | results["market_data"] = cleared
478 | total_cleared += cleared
479 |
480 | return {
481 | "success": True,
482 | "total_entries_cleared": total_cleared,
483 | "cleared_by_type": results,
484 | "timestamp": datetime.now().isoformat(),
485 | }
486 |
487 | except Exception as e:
488 | logger.error(f"Error clearing performance caches: {e}")
489 | return {
490 | "success": False,
491 | "error": str(e),
492 | "timestamp": datetime.now().isoformat(),
493 | }
494 |
```
--------------------------------------------------------------------------------
/maverick_mcp/api/middleware/rate_limiting_enhanced.py:
--------------------------------------------------------------------------------
```python
1 | """Enhanced rate limiting middleware and utilities."""
2 |
3 | from __future__ import annotations
4 |
5 | import asyncio
6 | import logging
7 | import time
8 | from collections import defaultdict, deque
9 | from collections.abc import Awaitable, Callable
10 | from dataclasses import dataclass
11 | from enum import Enum
12 | from functools import wraps
13 | from typing import Any
14 |
15 | from fastapi import Request
16 | from starlette.middleware.base import BaseHTTPMiddleware
17 | from starlette.responses import JSONResponse, Response
18 |
19 | from maverick_mcp.data.performance import redis_manager
20 | from maverick_mcp.exceptions import RateLimitError
21 |
22 | logger = logging.getLogger(__name__)
23 |
24 |
25 | class RateLimitStrategy(str, Enum):
26 | """Supported rate limiting strategies."""
27 |
28 | SLIDING_WINDOW = "sliding_window"
29 | TOKEN_BUCKET = "token_bucket"
30 | FIXED_WINDOW = "fixed_window"
31 |
32 |
33 | class RateLimitTier(str, Enum):
34 | """Logical tiers used to classify API endpoints."""
35 |
36 | PUBLIC = "public"
37 | AUTHENTICATION = "authentication"
38 | DATA_RETRIEVAL = "data_retrieval"
39 | ANALYSIS = "analysis"
40 | BULK_OPERATION = "bulk_operation"
41 | ADMINISTRATIVE = "administrative"
42 |
43 |
44 | class EndpointClassification:
45 | """Utility helpers for mapping endpoints to rate limit tiers."""
46 |
47 | @staticmethod
48 | def classify_endpoint(path: str) -> RateLimitTier:
49 | normalized = path.lower()
50 | if normalized in {
51 | "/health",
52 | "/metrics",
53 | "/docs",
54 | "/openapi.json",
55 | "/api/docs",
56 | "/api/openapi.json",
57 | }:
58 | return RateLimitTier.PUBLIC
59 | if normalized.startswith("/api/auth"):
60 | return RateLimitTier.AUTHENTICATION
61 | if "admin" in normalized:
62 | return RateLimitTier.ADMINISTRATIVE
63 | if "bulk" in normalized or normalized.endswith("/batch"):
64 | return RateLimitTier.BULK_OPERATION
65 | if any(
66 | segment in normalized
67 | for segment in ("analysis", "technical", "screening", "portfolio")
68 | ):
69 | return RateLimitTier.ANALYSIS
70 | return RateLimitTier.DATA_RETRIEVAL
71 |
72 |
73 | @dataclass(slots=True)
74 | class RateLimitConfig:
75 | """Configuration options for rate limiting."""
76 |
77 | public_limit: int = 100
78 | auth_limit: int = 30
79 | data_limit: int = 60
80 | data_limit_anonymous: int = 15
81 | analysis_limit: int = 30
82 | analysis_limit_anonymous: int = 10
83 | bulk_limit_per_hour: int = 10
84 | admin_limit: int = 20
85 | premium_multiplier: float = 3.0
86 | enterprise_multiplier: float = 5.0
87 | default_strategy: RateLimitStrategy = RateLimitStrategy.SLIDING_WINDOW
88 | burst_multiplier: float = 1.5
89 | window_size_seconds: int = 60
90 | token_refill_rate: float = 1.0
91 | max_tokens: int | None = None
92 | log_violations: bool = True
93 | alert_threshold: int = 5
94 |
95 | def limit_for(
96 | self, tier: RateLimitTier, *, authenticated: bool, role: str | None = None
97 | ) -> int:
98 | limit = self.data_limit
99 | if tier == RateLimitTier.PUBLIC:
100 | limit = self.public_limit
101 | elif tier == RateLimitTier.AUTHENTICATION:
102 | limit = self.auth_limit
103 | elif tier == RateLimitTier.DATA_RETRIEVAL:
104 | limit = self.data_limit if authenticated else self.data_limit_anonymous
105 | elif tier == RateLimitTier.ANALYSIS:
106 | limit = (
107 | self.analysis_limit if authenticated else self.analysis_limit_anonymous
108 | )
109 | elif tier == RateLimitTier.BULK_OPERATION:
110 | limit = self.bulk_limit_per_hour
111 | elif tier == RateLimitTier.ADMINISTRATIVE:
112 | limit = self.admin_limit
113 |
114 | normalized_role = (role or "").lower()
115 | if normalized_role == "premium":
116 | limit = int(limit * self.premium_multiplier)
117 | elif normalized_role == "enterprise":
118 | limit = int(limit * self.enterprise_multiplier)
119 |
120 | return max(limit, 1)
121 |
122 |
123 | class RateLimiter:
124 | """Core rate limiter that operates with Redis and an in-process fallback."""
125 |
126 | def __init__(self, config: RateLimitConfig) -> None:
127 | self.config = config
128 | self._local_counters: dict[str, deque[float]] = defaultdict(deque)
129 | self._violations: dict[str, int] = defaultdict(int)
130 |
131 | @staticmethod
132 | def _tiered_key(tier: RateLimitTier, identifier: str) -> str:
133 | """Compose a namespaced key for tracking tier-specific counters."""
134 |
135 | return f"{tier.value}:{identifier}"
136 |
137 | def _redis_key(self, prefix: str, *, tier: RateLimitTier, identifier: str) -> str:
138 | """Build a Redis key for the given strategy prefix and identifier."""
139 |
140 | tiered_identifier = self._tiered_key(tier, identifier)
141 | return f"rate_limit:{prefix}:{tiered_identifier}"
142 |
143 | async def check_rate_limit(
144 | self,
145 | *,
146 | key: str,
147 | tier: RateLimitTier,
148 | limit: int,
149 | window_seconds: int,
150 | strategy: RateLimitStrategy | None = None,
151 | ) -> tuple[bool, dict[str, Any]]:
152 | strategy = strategy or self.config.default_strategy
153 | client = await redis_manager.get_client()
154 | tiered_key = self._tiered_key(tier, key)
155 | if client is None:
156 | allowed, info = self._check_local_rate_limit(
157 | key=tiered_key,
158 | limit=limit,
159 | window_seconds=window_seconds,
160 | )
161 | info["strategy"] = strategy.value
162 | info["tier"] = tier.value
163 | info["fallback"] = True
164 | return allowed, info
165 |
166 | if strategy == RateLimitStrategy.SLIDING_WINDOW:
167 | return await self._check_sliding_window(
168 | client, key, tier, limit, window_seconds
169 | )
170 | if strategy == RateLimitStrategy.TOKEN_BUCKET:
171 | return await self._check_token_bucket(
172 | client, key, tier, limit, window_seconds
173 | )
174 | return await self._check_fixed_window(client, key, tier, limit, window_seconds)
175 |
176 | def record_violation(self, key: str, *, tier: RateLimitTier | None = None) -> None:
177 | namespaced_key = self._tiered_key(tier, key) if tier else key
178 | self._violations[namespaced_key] += 1
179 |
180 | def get_violation_count(
181 | self, key: str, *, tier: RateLimitTier | None = None
182 | ) -> int:
183 | namespaced_key = self._tiered_key(tier, key) if tier else key
184 | return self._violations.get(namespaced_key, 0)
185 |
186 | def _check_local_rate_limit(
187 | self,
188 | *,
189 | key: str,
190 | limit: int,
191 | window_seconds: int,
192 | ) -> tuple[bool, dict[str, Any]]:
193 | now = time.time()
194 | window_start = now - window_seconds
195 | bucket = self._local_counters[key]
196 | while bucket and bucket[0] <= window_start:
197 | bucket.popleft()
198 |
199 | if len(bucket) >= limit:
200 | retry_after = int(bucket[0] + window_seconds - now) + 1
201 | return False, {
202 | "limit": limit,
203 | "remaining": 0,
204 | "retry_after": max(retry_after, 1),
205 | }
206 |
207 | bucket.append(now)
208 | remaining = max(limit - len(bucket), 0)
209 | return True, {"limit": limit, "remaining": remaining}
210 |
211 | async def _check_sliding_window(
212 | self,
213 | client: Any,
214 | key: str,
215 | tier: RateLimitTier,
216 | limit: int,
217 | window_seconds: int,
218 | ) -> tuple[bool, dict[str, Any]]:
219 | redis_key = self._redis_key("sw", tier=tier, identifier=key)
220 | now = time.time()
221 |
222 | pipeline = client.pipeline()
223 | pipeline.zremrangebyscore(redis_key, 0, now - window_seconds)
224 | pipeline.zcard(redis_key)
225 | pipeline.zadd(redis_key, {str(now): now})
226 | pipeline.expire(redis_key, window_seconds)
227 | results = await pipeline.execute()
228 |
229 | current_count = int(results[1]) + 1
230 | remaining = max(limit - current_count, 0)
231 | info: dict[str, Any] = {
232 | "limit": limit,
233 | "remaining": remaining,
234 | "burst_limit": int(limit * self.config.burst_multiplier),
235 | "strategy": RateLimitStrategy.SLIDING_WINDOW.value,
236 | "tier": tier.value,
237 | }
238 |
239 | if current_count > limit:
240 | oldest = await client.zrange(redis_key, 0, 0, withscores=True)
241 | retry_after = 1
242 | if oldest:
243 | oldest_ts = float(oldest[0][1])
244 | retry_after = max(int(oldest_ts + window_seconds - now), 1)
245 | info["remaining"] = 0
246 | info["retry_after"] = retry_after
247 | return False, info
248 |
249 | return True, info
250 |
251 | async def _check_token_bucket(
252 | self,
253 | client: Any,
254 | key: str,
255 | tier: RateLimitTier,
256 | limit: int,
257 | window_seconds: int,
258 | ) -> tuple[bool, dict[str, Any]]:
259 | redis_key = self._redis_key("tb", tier=tier, identifier=key)
260 | now = time.time()
261 | state = await client.hgetall(redis_key)
262 |
263 | def _decode_value(mapping: dict[Any, Any], key: str) -> str | None:
264 | value = mapping.get(key)
265 | if value is None:
266 | value = mapping.get(key.encode("utf-8"))
267 | if isinstance(value, bytes):
268 | return value.decode("utf-8")
269 | return value
270 |
271 | if state:
272 | tokens_value = _decode_value(state, "tokens")
273 | last_refill_value = _decode_value(state, "last_refill")
274 | else:
275 | tokens_value = None
276 | last_refill_value = None
277 |
278 | tokens = float(tokens_value) if tokens_value is not None else float(limit)
279 | last_refill = float(last_refill_value) if last_refill_value is not None else now
280 | elapsed = max(now - last_refill, 0)
281 | capacity = float(limit)
282 | if self.config.max_tokens is not None:
283 | capacity = min(capacity, float(self.config.max_tokens))
284 | tokens = min(capacity, tokens + elapsed * self.config.token_refill_rate)
285 |
286 | info: dict[str, Any] = {
287 | "limit": limit,
288 | "tokens": tokens,
289 | "refill_rate": self.config.token_refill_rate,
290 | "strategy": RateLimitStrategy.TOKEN_BUCKET.value,
291 | "tier": tier.value,
292 | }
293 |
294 | if tokens < 1:
295 | retry_after = int(max((1 - tokens) / self.config.token_refill_rate, 1))
296 | info["remaining"] = 0
297 | info["retry_after"] = retry_after
298 | await client.hset(redis_key, mapping={"tokens": tokens, "last_refill": now})
299 | await client.expire(redis_key, window_seconds)
300 | return False, info
301 |
302 | tokens -= 1
303 | info["remaining"] = int(tokens)
304 | await client.hset(redis_key, mapping={"tokens": tokens, "last_refill": now})
305 | await client.expire(redis_key, window_seconds)
306 | return True, info
307 |
308 | async def _check_fixed_window(
309 | self,
310 | client: Any,
311 | key: str,
312 | tier: RateLimitTier,
313 | limit: int,
314 | window_seconds: int,
315 | ) -> tuple[bool, dict[str, Any]]:
316 | redis_key = self._redis_key("fw", tier=tier, identifier=key)
317 | pipeline = client.pipeline()
318 | pipeline.incr(redis_key)
319 | pipeline.expire(redis_key, window_seconds)
320 | results = await pipeline.execute()
321 |
322 | current = int(results[0])
323 | remaining = max(limit - current, 0)
324 | info = {
325 | "limit": limit,
326 | "current_count": current,
327 | "remaining": remaining,
328 | "strategy": RateLimitStrategy.FIXED_WINDOW.value,
329 | "tier": tier.value,
330 | }
331 |
332 | if current > limit:
333 | info["retry_after"] = window_seconds
334 | info["remaining"] = 0
335 | return False, info
336 |
337 | return True, info
338 |
339 | async def cleanup_old_data(self, *, older_than_hours: int = 24) -> None:
340 | client = await redis_manager.get_client()
341 | if client is None:
342 | return
343 |
344 | cutoff = time.time() - older_than_hours * 3600
345 | cursor = 0
346 | while True:
347 | cursor, keys = await client.scan(
348 | cursor=cursor, match="rate_limit:*", count=200
349 | )
350 | for raw_key in keys:
351 | key = (
352 | raw_key.decode()
353 | if isinstance(raw_key, bytes | bytearray)
354 | else raw_key
355 | )
356 | redis_type = await client.type(key)
357 | if redis_type == "zset":
358 | await client.zremrangebyscore(key, 0, cutoff)
359 | if await client.zcard(key) == 0:
360 | await client.delete(key)
361 | elif redis_type == "string":
362 | ttl = await client.ttl(key)
363 | if ttl == -1:
364 | await client.expire(key, int(older_than_hours * 3600))
365 | if cursor == 0:
366 | break
367 |
368 |
369 | class EnhancedRateLimitMiddleware(BaseHTTPMiddleware):
370 | """FastAPI middleware that enforces rate limits for each request."""
371 |
372 | def __init__(self, app, config: RateLimitConfig | None = None) -> None: # type: ignore[override]
373 | super().__init__(app)
374 | self.config = config or RateLimitConfig()
375 | self.rate_limiter = RateLimiter(self.config)
376 |
377 | async def dispatch(
378 | self, request: Request, call_next: Callable[[Request], Awaitable[Response]]
379 | ) -> Response: # type: ignore[override]
380 | path = request.url.path
381 | tier = EndpointClassification.classify_endpoint(path)
382 | user_id = getattr(request.state, "user_id", None)
383 | user_context = getattr(request.state, "user_context", {}) or {}
384 | role = user_context.get("role") if isinstance(user_context, dict) else None
385 | authenticated = bool(user_id)
386 | client = getattr(request, "client", None)
387 | client_host = getattr(client, "host", None) if client else None
388 | key = str(user_id or client_host or "anonymous")
389 |
390 | limit = self.config.limit_for(tier, authenticated=authenticated, role=role)
391 | allowed, info = await self.rate_limiter.check_rate_limit(
392 | key=key,
393 | tier=tier,
394 | limit=limit,
395 | window_seconds=self.config.window_size_seconds,
396 | )
397 |
398 | if not allowed:
399 | retry_after = int(info.get("retry_after", 1))
400 | self.rate_limiter.record_violation(key, tier=tier)
401 | if self.config.log_violations:
402 | logger.warning("Rate limit exceeded for %s (%s)", key, tier.value)
403 | error = RateLimitError(retry_after=retry_after, context={"info": info})
404 | headers = {"Retry-After": str(retry_after)}
405 | body = {"error": error.message}
406 | return JSONResponse(body, status_code=error.status_code, headers=headers)
407 |
408 | response = await call_next(request)
409 | response.headers["X-RateLimit-Limit"] = str(limit)
410 | response.headers["X-RateLimit-Remaining"] = str(
411 | max(info.get("remaining", limit), 0)
412 | )
413 | response.headers["X-RateLimit-Tier"] = tier.value
414 | return response
415 |
416 |
417 | _default_config = RateLimitConfig()
418 | _default_limiter = RateLimiter(_default_config)
419 |
420 |
421 | def _extract_request(args: tuple[Any, ...], kwargs: dict[str, Any]) -> Request | None:
422 | for value in list(args) + list(kwargs.values()):
423 | if isinstance(value, Request):
424 | return value
425 | if hasattr(value, "state") and hasattr(value, "url"):
426 | return value # type: ignore[return-value]
427 | return None
428 |
429 |
430 | def rate_limit(
431 | *,
432 | requests_per_minute: int,
433 | strategy: RateLimitStrategy | None = None,
434 | ) -> Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]:
435 | """Function decorator enforcing rate limits for arbitrary callables."""
436 |
437 | window_seconds = 60
438 |
439 | def decorator(func: Callable[..., Awaitable[Any]]):
440 | if not asyncio.iscoroutinefunction(func):
441 | raise TypeError(
442 | "rate_limit decorator can only be applied to async callables"
443 | )
444 |
445 | @wraps(func)
446 | async def wrapper(*args: Any, **kwargs: Any) -> Any:
447 | request = _extract_request(args, kwargs)
448 | if request is None:
449 | return await func(*args, **kwargs)
450 |
451 | tier = EndpointClassification.classify_endpoint(request.url.path)
452 | key = str(getattr(request.state, "user_id", None) or request.url.path)
453 | allowed, info = await _default_limiter.check_rate_limit(
454 | key=key,
455 | tier=tier,
456 | limit=requests_per_minute,
457 | window_seconds=window_seconds,
458 | strategy=strategy,
459 | )
460 | if not allowed:
461 | retry_after = int(info.get("retry_after", 1))
462 | raise RateLimitError(retry_after=retry_after, context={"info": info})
463 |
464 | return await func(*args, **kwargs)
465 |
466 | return wrapper
467 |
468 | return decorator
469 |
```
--------------------------------------------------------------------------------
/maverick_mcp/api/routers/screening_ddd.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | DDD-based screening router for Maverick-MCP.
3 |
4 | This module demonstrates Domain-Driven Design principles with clear
5 | separation between layers and dependency injection.
6 | """
7 |
8 | import logging
9 | import time
10 | from datetime import datetime
11 | from decimal import Decimal
12 | from typing import Any
13 |
14 | from fastapi import Depends
15 | from fastmcp import FastMCP
16 | from pydantic import BaseModel, Field
17 |
18 | # Application layer imports
19 | from maverick_mcp.application.screening.queries import (
20 | GetAllScreeningResultsQuery,
21 | GetScreeningResultsQuery,
22 | GetScreeningStatisticsQuery,
23 | )
24 | from maverick_mcp.domain.screening.services import IStockRepository
25 |
26 | # Domain layer imports
27 | from maverick_mcp.domain.screening.value_objects import (
28 | ScreeningCriteria,
29 | ScreeningStrategy,
30 | )
31 |
32 | # Infrastructure layer imports
33 | from maverick_mcp.infrastructure.screening.repositories import (
34 | CachedStockRepository,
35 | PostgresStockRepository,
36 | )
37 |
38 | logger = logging.getLogger(__name__)
39 |
40 | # Create the DDD screening router
41 | screening_ddd_router: FastMCP = FastMCP("Stock_Screening_DDD")
42 |
43 |
44 | # Dependency Injection Setup
45 | def get_stock_repository() -> IStockRepository:
46 | """
47 | Dependency injection for stock repository.
48 |
49 | This function provides the concrete repository implementation
50 | with caching capabilities.
51 | """
52 | base_repository = PostgresStockRepository()
53 | cached_repository = CachedStockRepository(base_repository, cache_ttl_seconds=300)
54 | return cached_repository
55 |
56 |
57 | # Request/Response Models for MCP Tools
58 | class ScreeningRequest(BaseModel):
59 | """Request model for screening operations."""
60 |
61 | strategy: str = Field(
62 | description="Screening strategy (maverick_bullish, maverick_bearish, trending_stage2)"
63 | )
64 | limit: int = Field(
65 | default=20, ge=1, le=100, description="Maximum number of results"
66 | )
67 | min_momentum_score: float | None = Field(
68 | default=None, ge=0, le=100, description="Minimum momentum score"
69 | )
70 | min_volume: int | None = Field(
71 | default=None, ge=0, description="Minimum daily volume"
72 | )
73 | max_price: float | None = Field(
74 | default=None, gt=0, description="Maximum stock price"
75 | )
76 | min_price: float | None = Field(
77 | default=None, gt=0, description="Minimum stock price"
78 | )
79 | require_above_sma50: bool = Field(
80 | default=False, description="Require price above SMA 50"
81 | )
82 | require_pattern_detected: bool = Field(
83 | default=False, description="Require pattern detection"
84 | )
85 |
86 |
87 | class AllScreeningRequest(BaseModel):
88 | """Request model for all screening strategies."""
89 |
90 | limit_per_strategy: int = Field(
91 | default=10, ge=1, le=50, description="Results per strategy"
92 | )
93 | min_momentum_score: float | None = Field(
94 | default=None, ge=0, le=100, description="Minimum momentum score filter"
95 | )
96 |
97 |
98 | class StatisticsRequest(BaseModel):
99 | """Request model for screening statistics."""
100 |
101 | strategy: str | None = Field(
102 | default=None, description="Specific strategy to analyze (None for all)"
103 | )
104 | limit: int = Field(
105 | default=100, ge=1, le=500, description="Maximum results to analyze"
106 | )
107 |
108 |
109 | # Helper Functions
110 | def _create_screening_criteria_from_request(
111 | request: ScreeningRequest,
112 | ) -> ScreeningCriteria:
113 | """Convert API request to domain value object."""
114 | return ScreeningCriteria(
115 | min_momentum_score=Decimal(str(request.min_momentum_score))
116 | if request.min_momentum_score
117 | else None,
118 | min_volume=request.min_volume,
119 | min_price=Decimal(str(request.min_price)) if request.min_price else None,
120 | max_price=Decimal(str(request.max_price)) if request.max_price else None,
121 | require_above_sma50=request.require_above_sma50,
122 | require_pattern_detected=request.require_pattern_detected,
123 | )
124 |
125 |
126 | def _convert_collection_to_dto(
127 | collection, execution_time_ms: float, applied_filters: dict[str, Any]
128 | ) -> dict[str, Any]:
129 | """Convert domain collection to API response DTO."""
130 | results_dto = []
131 | for result in collection.results:
132 | result_dict = result.to_dict()
133 |
134 | # Convert domain result to DTO format
135 | result_dto = {
136 | "stock_symbol": result_dict["stock_symbol"],
137 | "screening_date": result_dict["screening_date"],
138 | "close_price": result_dict["close_price"],
139 | "volume": result.volume,
140 | "momentum_score": result_dict["momentum_score"],
141 | "adr_percentage": result_dict["adr_percentage"],
142 | "ema_21": float(result.ema_21),
143 | "sma_50": float(result.sma_50),
144 | "sma_150": float(result.sma_150),
145 | "sma_200": float(result.sma_200),
146 | "avg_volume_30d": float(result.avg_volume_30d),
147 | "atr": float(result.atr),
148 | "pattern": result.pattern,
149 | "squeeze": result.squeeze,
150 | "consolidation": result.vcp,
151 | "entry_signal": result.entry_signal,
152 | "combined_score": result.combined_score,
153 | "bear_score": result.bear_score,
154 | "quality_score": result_dict["quality_score"],
155 | "is_bullish": result_dict["is_bullish"],
156 | "is_bearish": result_dict["is_bearish"],
157 | "is_trending_stage2": result_dict["is_trending_stage2"],
158 | "risk_reward_ratio": result_dict["risk_reward_ratio"],
159 | # Bearish-specific fields
160 | "rsi_14": float(result.rsi_14) if result.rsi_14 else None,
161 | "macd": float(result.macd) if result.macd else None,
162 | "macd_signal": float(result.macd_signal) if result.macd_signal else None,
163 | "macd_histogram": float(result.macd_histogram)
164 | if result.macd_histogram
165 | else None,
166 | "distribution_days_20": result.distribution_days_20,
167 | "atr_contraction": result.atr_contraction,
168 | "big_down_volume": result.big_down_volume,
169 | }
170 | results_dto.append(result_dto)
171 |
172 | return {
173 | "strategy_used": collection.strategy_used,
174 | "screening_timestamp": collection.screening_timestamp.isoformat(),
175 | "total_candidates_analyzed": collection.total_candidates_analyzed,
176 | "results_returned": len(collection.results),
177 | "results": results_dto,
178 | "statistics": collection.get_statistics(),
179 | "applied_filters": applied_filters,
180 | "sorting_applied": {"field": "strategy_default", "descending": True},
181 | "status": "success",
182 | "execution_time_ms": execution_time_ms,
183 | "warnings": [],
184 | }
185 |
186 |
187 | # MCP Tools
188 | @screening_ddd_router.tool()
189 | async def get_screening_results_ddd(
190 | request: ScreeningRequest,
191 | repository: IStockRepository = Depends(get_stock_repository),
192 | ) -> dict[str, Any]:
193 | """
194 | Get stock screening results using Domain-Driven Design architecture.
195 |
196 | This tool demonstrates DDD principles with clean separation of concerns:
197 | - Domain layer: Pure business logic and rules
198 | - Application layer: Orchestration and use cases
199 | - Infrastructure layer: Data access and external services
200 | - API layer: Request/response handling with dependency injection
201 |
202 | Args:
203 | request: Screening parameters including strategy and filters
204 | repository: Injected repository dependency
205 |
206 | Returns:
207 | Comprehensive screening results with business intelligence
208 | """
209 | start_time = time.time()
210 |
211 | try:
212 | # Validate strategy
213 | try:
214 | strategy = ScreeningStrategy(request.strategy)
215 | except ValueError:
216 | return {
217 | "status": "error",
218 | "error_code": "INVALID_STRATEGY",
219 | "error_message": f"Invalid strategy: {request.strategy}",
220 | "valid_strategies": [s.value for s in ScreeningStrategy],
221 | "timestamp": datetime.utcnow().isoformat(),
222 | }
223 |
224 | # Convert request to domain value objects
225 | criteria = _create_screening_criteria_from_request(request)
226 |
227 | # Execute application query
228 | query = GetScreeningResultsQuery(repository)
229 | collection = await query.execute(
230 | strategy=strategy, limit=request.limit, criteria=criteria
231 | )
232 |
233 | # Calculate execution time
234 | execution_time_ms = (time.time() - start_time) * 1000
235 |
236 | # Convert to API response
237 | applied_filters = {
238 | "strategy": request.strategy,
239 | "limit": request.limit,
240 | "min_momentum_score": request.min_momentum_score,
241 | "min_volume": request.min_volume,
242 | "min_price": request.min_price,
243 | "max_price": request.max_price,
244 | "require_above_sma50": request.require_above_sma50,
245 | "require_pattern_detected": request.require_pattern_detected,
246 | }
247 |
248 | response = _convert_collection_to_dto(
249 | collection, execution_time_ms, applied_filters
250 | )
251 |
252 | logger.info(
253 | f"DDD screening completed: {strategy.value}, "
254 | f"{len(collection.results)} results, {execution_time_ms:.1f}ms"
255 | )
256 |
257 | return response
258 |
259 | except Exception as e:
260 | execution_time_ms = (time.time() - start_time) * 1000
261 | logger.error(f"Error in DDD screening: {e}")
262 |
263 | return {
264 | "status": "error",
265 | "error_code": "SCREENING_FAILED",
266 | "error_message": str(e),
267 | "execution_time_ms": execution_time_ms,
268 | "timestamp": datetime.utcnow().isoformat(),
269 | }
270 |
271 |
272 | @screening_ddd_router.tool()
273 | async def get_all_screening_results_ddd(
274 | request: AllScreeningRequest,
275 | repository: IStockRepository = Depends(get_stock_repository),
276 | ) -> dict[str, Any]:
277 | """
278 | Get screening results from all strategies using DDD architecture.
279 |
280 | This tool executes all available screening strategies and provides
281 | comprehensive cross-strategy analysis and insights.
282 |
283 | Args:
284 | request: Parameters for multi-strategy screening
285 | repository: Injected repository dependency
286 |
287 | Returns:
288 | Results from all strategies with cross-strategy analysis
289 | """
290 | start_time = time.time()
291 |
292 | try:
293 | # Create criteria if filters provided
294 | criteria = None
295 | if request.min_momentum_score:
296 | criteria = ScreeningCriteria(
297 | min_momentum_score=Decimal(str(request.min_momentum_score))
298 | )
299 |
300 | # Execute application query
301 | query = GetAllScreeningResultsQuery(repository)
302 | all_collections = await query.execute(
303 | limit_per_strategy=request.limit_per_strategy, criteria=criteria
304 | )
305 |
306 | # Calculate execution time
307 | execution_time_ms = (time.time() - start_time) * 1000
308 |
309 | # Convert collections to DTOs
310 | response = {
311 | "screening_timestamp": datetime.utcnow().isoformat(),
312 | "strategies_executed": list(all_collections.keys()),
313 | "execution_time_ms": execution_time_ms,
314 | "status": "success",
315 | "errors": [],
316 | }
317 |
318 | # Add individual strategy results
319 | for strategy_name, collection in all_collections.items():
320 | applied_filters = {"limit": request.limit_per_strategy}
321 | if request.min_momentum_score:
322 | applied_filters["min_momentum_score"] = request.min_momentum_score
323 |
324 | strategy_dto = _convert_collection_to_dto(
325 | collection,
326 | execution_time_ms
327 | / len(all_collections), # Approximate per-strategy time
328 | applied_filters,
329 | )
330 |
331 | # Map strategy names to response fields
332 | if strategy_name == ScreeningStrategy.MAVERICK_BULLISH.value:
333 | response["maverick_bullish"] = strategy_dto
334 | elif strategy_name == ScreeningStrategy.MAVERICK_BEARISH.value:
335 | response["maverick_bearish"] = strategy_dto
336 | elif strategy_name == ScreeningStrategy.TRENDING_STAGE2.value:
337 | response["trending_stage2"] = strategy_dto
338 |
339 | # Add cross-strategy analysis
340 | statistics_query = GetScreeningStatisticsQuery(repository)
341 | stats = await statistics_query.execute(limit=request.limit_per_strategy * 3)
342 |
343 | response["cross_strategy_analysis"] = stats.get("cross_strategy_analysis", {})
344 | response["overall_summary"] = stats.get("overall_summary", {})
345 |
346 | logger.info(
347 | f"DDD all screening completed: {len(all_collections)} strategies, "
348 | f"{execution_time_ms:.1f}ms"
349 | )
350 |
351 | return response
352 |
353 | except Exception as e:
354 | execution_time_ms = (time.time() - start_time) * 1000
355 | logger.error(f"Error in DDD all screening: {e}")
356 |
357 | return {
358 | "screening_timestamp": datetime.utcnow().isoformat(),
359 | "strategies_executed": [],
360 | "status": "error",
361 | "error_code": "ALL_SCREENING_FAILED",
362 | "error_message": str(e),
363 | "execution_time_ms": execution_time_ms,
364 | "errors": [str(e)],
365 | }
366 |
367 |
368 | @screening_ddd_router.tool()
369 | async def get_screening_statistics_ddd(
370 | request: StatisticsRequest,
371 | repository: IStockRepository = Depends(get_stock_repository),
372 | ) -> dict[str, Any]:
373 | """
374 | Get comprehensive screening statistics and analytics using DDD architecture.
375 |
376 | This tool provides business intelligence and analytical insights
377 | for screening operations, demonstrating how domain services can
378 | calculate complex business metrics.
379 |
380 | Args:
381 | request: Statistics parameters
382 | repository: Injected repository dependency
383 |
384 | Returns:
385 | Comprehensive statistics and business intelligence
386 | """
387 | start_time = time.time()
388 |
389 | try:
390 | # Validate strategy if provided
391 | strategy = None
392 | if request.strategy:
393 | try:
394 | strategy = ScreeningStrategy(request.strategy)
395 | except ValueError:
396 | return {
397 | "status": "error",
398 | "error_code": "INVALID_STRATEGY",
399 | "error_message": f"Invalid strategy: {request.strategy}",
400 | "valid_strategies": [s.value for s in ScreeningStrategy],
401 | "timestamp": datetime.utcnow().isoformat(),
402 | }
403 |
404 | # Execute statistics query
405 | query = GetScreeningStatisticsQuery(repository)
406 | stats = await query.execute(strategy=strategy, limit=request.limit)
407 |
408 | # Calculate execution time
409 | execution_time_ms = (time.time() - start_time) * 1000
410 |
411 | # Enhance response with metadata
412 | stats.update(
413 | {
414 | "execution_time_ms": execution_time_ms,
415 | "status": "success",
416 | "analysis_scope": "single" if strategy else "all",
417 | "results_analyzed": request.limit,
418 | }
419 | )
420 |
421 | logger.info(
422 | f"DDD statistics completed: {strategy.value if strategy else 'all'}, "
423 | f"{execution_time_ms:.1f}ms"
424 | )
425 |
426 | return stats
427 |
428 | except Exception as e:
429 | execution_time_ms = (time.time() - start_time) * 1000
430 | logger.error(f"Error in DDD statistics: {e}")
431 |
432 | return {
433 | "status": "error",
434 | "error_code": "STATISTICS_FAILED",
435 | "error_message": str(e),
436 | "execution_time_ms": execution_time_ms,
437 | "timestamp": datetime.utcnow().isoformat(),
438 | "analysis_scope": "failed",
439 | "results_analyzed": 0,
440 | }
441 |
442 |
443 | @screening_ddd_router.tool()
444 | async def get_repository_cache_stats(
445 | repository: IStockRepository = Depends(get_stock_repository),
446 | ) -> dict[str, Any]:
447 | """
448 | Get repository cache statistics for monitoring and optimization.
449 |
450 | This tool demonstrates infrastructure layer monitoring capabilities
451 | and provides insights into caching performance.
452 |
453 | Args:
454 | repository: Injected repository dependency
455 |
456 | Returns:
457 | Cache statistics and performance metrics
458 | """
459 | try:
460 | # Check if repository supports cache statistics
461 | if hasattr(repository, "get_cache_stats"):
462 | cache_stats = repository.get_cache_stats()
463 |
464 | return {
465 | "status": "success",
466 | "cache_enabled": True,
467 | "cache_statistics": cache_stats,
468 | "timestamp": datetime.utcnow().isoformat(),
469 | }
470 | else:
471 | return {
472 | "status": "success",
473 | "cache_enabled": False,
474 | "message": "Repository does not support caching",
475 | "timestamp": datetime.utcnow().isoformat(),
476 | }
477 |
478 | except Exception as e:
479 | logger.error(f"Error getting cache stats: {e}")
480 |
481 | return {
482 | "status": "error",
483 | "error_message": str(e),
484 | "timestamp": datetime.utcnow().isoformat(),
485 | }
486 |
```
--------------------------------------------------------------------------------
/maverick_mcp/workflows/agents/strategy_selector.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Strategy Selector Agent for intelligent strategy recommendation.
3 |
4 | This agent analyzes market regime and selects the most appropriate trading strategies
5 | based on current market conditions, volatility, and trend characteristics.
6 | """
7 |
8 | import logging
9 | from datetime import datetime
10 | from typing import Any
11 |
12 | from maverick_mcp.backtesting.strategies.templates import (
13 | get_strategy_info,
14 | list_available_strategies,
15 | )
16 | from maverick_mcp.workflows.state import BacktestingWorkflowState
17 |
18 | logger = logging.getLogger(__name__)
19 |
20 |
21 | class StrategySelectorAgent:
22 | """Intelligent strategy selector based on market regime analysis."""
23 |
24 | def __init__(self):
25 | """Initialize strategy selector agent."""
26 | # Strategy fitness mapping for different market regimes
27 | self.REGIME_STRATEGY_FITNESS = {
28 | "trending": {
29 | "sma_cross": 0.9,
30 | "ema_cross": 0.85,
31 | "macd": 0.8,
32 | "breakout": 0.9,
33 | "momentum": 0.85,
34 | "rsi": 0.3, # Poor for trending markets
35 | "bollinger": 0.4,
36 | "mean_reversion": 0.2,
37 | },
38 | "ranging": {
39 | "rsi": 0.9,
40 | "bollinger": 0.85,
41 | "mean_reversion": 0.9,
42 | "sma_cross": 0.3, # Poor for ranging markets
43 | "ema_cross": 0.3,
44 | "breakout": 0.2,
45 | "momentum": 0.25,
46 | "macd": 0.5,
47 | },
48 | "volatile": {
49 | "breakout": 0.8,
50 | "momentum": 0.7,
51 | "volatility_breakout": 0.9,
52 | "bollinger": 0.7,
53 | "sma_cross": 0.4,
54 | "rsi": 0.6,
55 | "mean_reversion": 0.5,
56 | "macd": 0.5,
57 | },
58 | "volatile_trending": {
59 | "breakout": 0.85,
60 | "momentum": 0.8,
61 | "volatility_breakout": 0.9,
62 | "macd": 0.7,
63 | "ema_cross": 0.6,
64 | "sma_cross": 0.6,
65 | "rsi": 0.4,
66 | "bollinger": 0.6,
67 | },
68 | "low_volume": {
69 | "sma_cross": 0.7,
70 | "ema_cross": 0.7,
71 | "rsi": 0.6,
72 | "mean_reversion": 0.6,
73 | "breakout": 0.3, # Poor for low volume
74 | "momentum": 0.4,
75 | "bollinger": 0.6,
76 | "macd": 0.6,
77 | },
78 | "low_volume_ranging": {
79 | "rsi": 0.8,
80 | "mean_reversion": 0.8,
81 | "bollinger": 0.7,
82 | "sma_cross": 0.5,
83 | "ema_cross": 0.5,
84 | "breakout": 0.2,
85 | "momentum": 0.3,
86 | "macd": 0.4,
87 | },
88 | "unknown": {
89 | # Balanced approach for unknown regimes
90 | "sma_cross": 0.6,
91 | "ema_cross": 0.6,
92 | "rsi": 0.6,
93 | "macd": 0.6,
94 | "bollinger": 0.6,
95 | "momentum": 0.5,
96 | "breakout": 0.5,
97 | "mean_reversion": 0.5,
98 | },
99 | }
100 |
101 | # Additional fitness adjustments based on market conditions
102 | self.CONDITION_ADJUSTMENTS = {
103 | "high_volatility": {
104 | "rsi": -0.1,
105 | "breakout": 0.1,
106 | "volatility_breakout": 0.15,
107 | },
108 | "low_volatility": {
109 | "mean_reversion": 0.1,
110 | "rsi": 0.1,
111 | "breakout": -0.1,
112 | },
113 | "high_volume": {
114 | "breakout": 0.1,
115 | "momentum": 0.1,
116 | "sma_cross": 0.05,
117 | },
118 | "low_volume": {
119 | "breakout": -0.15,
120 | "momentum": -0.1,
121 | "mean_reversion": 0.05,
122 | },
123 | }
124 |
125 | logger.info("StrategySelectorAgent initialized")
126 |
127 | async def select_strategies(
128 | self, state: BacktestingWorkflowState
129 | ) -> BacktestingWorkflowState:
130 | """Select optimal strategies based on market regime analysis.
131 |
132 | Args:
133 | state: Current workflow state with market regime analysis
134 |
135 | Returns:
136 | Updated state with strategy selection results
137 | """
138 | try:
139 | logger.info(
140 | f"Selecting strategies for {state['symbol']} in {state['market_regime']} regime"
141 | )
142 |
143 | # Get available strategies
144 | available_strategies = list_available_strategies()
145 |
146 | # Calculate strategy fitness scores
147 | strategy_rankings = self._calculate_strategy_fitness(
148 | state["market_regime"],
149 | state["market_conditions"],
150 | available_strategies,
151 | state["regime_confidence"],
152 | )
153 |
154 | # Select top strategies
155 | selected_strategies = self._select_top_strategies(
156 | strategy_rankings,
157 | user_preference=state["requested_strategy"],
158 | max_strategies=5, # Limit to top 5 for optimization efficiency
159 | )
160 |
161 | # Generate strategy candidates with metadata
162 | candidates = self._generate_strategy_candidates(
163 | selected_strategies, available_strategies
164 | )
165 |
166 | # Create selection reasoning
167 | reasoning = self._generate_selection_reasoning(
168 | state["market_regime"],
169 | state["regime_confidence"],
170 | selected_strategies,
171 | state["market_conditions"],
172 | )
173 |
174 | # Calculate selection confidence
175 | selection_confidence = self._calculate_selection_confidence(
176 | strategy_rankings,
177 | selected_strategies,
178 | state["regime_confidence"],
179 | )
180 |
181 | # Update state
182 | state["candidate_strategies"] = candidates
183 | state["strategy_rankings"] = strategy_rankings
184 | state["selected_strategies"] = selected_strategies
185 | state["strategy_selection_reasoning"] = reasoning
186 | state["strategy_selection_confidence"] = selection_confidence
187 |
188 | # Update workflow status
189 | state["workflow_status"] = "optimizing"
190 | state["current_step"] = "strategy_selection_completed"
191 | state["steps_completed"].append("strategy_selection")
192 |
193 | logger.info(
194 | f"Strategy selection completed for {state['symbol']}: "
195 | f"Selected {len(selected_strategies)} strategies with confidence {selection_confidence:.2f}"
196 | )
197 |
198 | return state
199 |
200 | except Exception as e:
201 | error_info = {
202 | "step": "strategy_selection",
203 | "error": str(e),
204 | "timestamp": datetime.now().isoformat(),
205 | "symbol": state["symbol"],
206 | }
207 | state["errors_encountered"].append(error_info)
208 |
209 | # Fallback to basic strategy set
210 | fallback_strategies = ["sma_cross", "rsi", "macd"]
211 | state["selected_strategies"] = fallback_strategies
212 | state["strategy_selection_confidence"] = 0.3
213 | state["fallback_strategies_used"].append("strategy_selection_fallback")
214 |
215 | logger.error(f"Strategy selection failed for {state['symbol']}: {e}")
216 | return state
217 |
218 | def _calculate_strategy_fitness(
219 | self,
220 | regime: str,
221 | market_conditions: dict[str, Any],
222 | available_strategies: list[str],
223 | regime_confidence: float,
224 | ) -> dict[str, float]:
225 | """Calculate fitness scores for all available strategies."""
226 | fitness_scores = {}
227 |
228 | # Base fitness from regime mapping
229 | base_fitness = self.REGIME_STRATEGY_FITNESS.get(
230 | regime, self.REGIME_STRATEGY_FITNESS["unknown"]
231 | )
232 |
233 | for strategy in available_strategies:
234 | # Start with base fitness score
235 | score = base_fitness.get(strategy, 0.5) # Default to neutral if not defined
236 |
237 | # Apply condition-based adjustments
238 | score = self._apply_condition_adjustments(
239 | score, strategy, market_conditions
240 | )
241 |
242 | # Weight by regime confidence
243 | # If low confidence, move scores toward neutral (0.5)
244 | confidence_weight = regime_confidence
245 | score = score * confidence_weight + 0.5 * (1 - confidence_weight)
246 |
247 | # Ensure score is in valid range
248 | fitness_scores[strategy] = max(0.0, min(1.0, score))
249 |
250 | return fitness_scores
251 |
252 | def _apply_condition_adjustments(
253 | self, base_score: float, strategy: str, market_conditions: dict[str, Any]
254 | ) -> float:
255 | """Apply market condition adjustments to base fitness score."""
256 | score = base_score
257 |
258 | # Get relevant conditions
259 | volatility_regime = market_conditions.get(
260 | "volatility_regime", "medium_volatility"
261 | )
262 | volume_regime = market_conditions.get("volume_regime", "normal_volume")
263 |
264 | # Apply volatility adjustments
265 | if volatility_regime in self.CONDITION_ADJUSTMENTS:
266 | adjustment = self.CONDITION_ADJUSTMENTS[volatility_regime].get(strategy, 0)
267 | score += adjustment
268 |
269 | # Apply volume adjustments
270 | if volume_regime in self.CONDITION_ADJUSTMENTS:
271 | adjustment = self.CONDITION_ADJUSTMENTS[volume_regime].get(strategy, 0)
272 | score += adjustment
273 |
274 | return score
275 |
276 | def _select_top_strategies(
277 | self,
278 | strategy_rankings: dict[str, float],
279 | user_preference: str | None = None,
280 | max_strategies: int = 5,
281 | ) -> list[str]:
282 | """Select top strategies based on fitness scores and user preferences."""
283 | # Sort strategies by fitness score
284 | sorted_strategies = sorted(
285 | strategy_rankings.items(), key=lambda x: x[1], reverse=True
286 | )
287 |
288 | selected = []
289 |
290 | # Always include user preference if specified and available
291 | if user_preference and user_preference in strategy_rankings:
292 | selected.append(user_preference)
293 | logger.info(f"Including user-requested strategy: {user_preference}")
294 |
295 | # Add top strategies up to limit
296 | for strategy, score in sorted_strategies:
297 | if len(selected) >= max_strategies:
298 | break
299 | if strategy not in selected and score > 0.4: # Minimum threshold
300 | selected.append(strategy)
301 |
302 | # Ensure we have at least 2 strategies
303 | if len(selected) < 2:
304 | for strategy, _ in sorted_strategies:
305 | if strategy not in selected:
306 | selected.append(strategy)
307 | if len(selected) >= 2:
308 | break
309 |
310 | return selected
311 |
312 | def _generate_strategy_candidates(
313 | self, selected_strategies: list[str], available_strategies: list[str]
314 | ) -> list[dict[str, Any]]:
315 | """Generate detailed candidate information for selected strategies."""
316 | candidates = []
317 |
318 | for strategy in selected_strategies:
319 | if strategy in available_strategies:
320 | strategy_info = get_strategy_info(strategy)
321 | candidates.append(
322 | {
323 | "strategy": strategy,
324 | "name": strategy_info.get("name", strategy.title()),
325 | "description": strategy_info.get("description", ""),
326 | "category": strategy_info.get("category", "unknown"),
327 | "parameters": strategy_info.get("parameters", {}),
328 | "risk_level": strategy_info.get("risk_level", "medium"),
329 | "best_market_conditions": strategy_info.get(
330 | "best_conditions", []
331 | ),
332 | }
333 | )
334 |
335 | return candidates
336 |
337 | def _generate_selection_reasoning(
338 | self,
339 | regime: str,
340 | regime_confidence: float,
341 | selected_strategies: list[str],
342 | market_conditions: dict[str, Any],
343 | ) -> str:
344 | """Generate human-readable reasoning for strategy selection."""
345 | reasoning_parts = []
346 |
347 | # Market regime reasoning
348 | reasoning_parts.append(
349 | f"Market regime identified as '{regime}' with {regime_confidence:.1%} confidence."
350 | )
351 |
352 | # Strategy selection reasoning
353 | if regime == "trending":
354 | reasoning_parts.append(
355 | "In trending markets, trend-following strategies like moving average crossovers "
356 | "and momentum strategies typically perform well."
357 | )
358 | elif regime == "ranging":
359 | reasoning_parts.append(
360 | "In ranging markets, mean-reversion strategies like RSI and Bollinger Bands "
361 | "are favored as they capitalize on price oscillations within a range."
362 | )
363 | elif regime == "volatile":
364 | reasoning_parts.append(
365 | "In volatile markets, breakout strategies and volatility-based approaches "
366 | "can capture large price movements effectively."
367 | )
368 |
369 | # Condition-specific reasoning
370 | volatility_regime = market_conditions.get("volatility_regime", "")
371 | if volatility_regime == "high_volatility":
372 | reasoning_parts.append(
373 | "High volatility conditions favor strategies that can handle larger price swings."
374 | )
375 | elif volatility_regime == "low_volatility":
376 | reasoning_parts.append(
377 | "Low volatility conditions favor mean-reversion and range-bound strategies."
378 | )
379 |
380 | volume_regime = market_conditions.get("volume_regime", "")
381 | if volume_regime == "low_volume":
382 | reasoning_parts.append(
383 | "Low volume conditions reduce reliability of breakout strategies and favor "
384 | "trend-following approaches with longer timeframes."
385 | )
386 |
387 | # Selected strategies summary
388 | reasoning_parts.append(
389 | f"Selected strategies: {', '.join(selected_strategies)} "
390 | f"based on their historical performance in similar market conditions."
391 | )
392 |
393 | return " ".join(reasoning_parts)
394 |
395 | def _calculate_selection_confidence(
396 | self,
397 | strategy_rankings: dict[str, float],
398 | selected_strategies: list[str],
399 | regime_confidence: float,
400 | ) -> float:
401 | """Calculate confidence in strategy selection."""
402 | if not selected_strategies or not strategy_rankings:
403 | return 0.0
404 |
405 | # Average fitness of selected strategies
406 | selected_scores = [strategy_rankings.get(s, 0.5) for s in selected_strategies]
407 | avg_selected_fitness = sum(selected_scores) / len(selected_scores)
408 |
409 | # Score spread (higher spread = more confident in selection)
410 | all_scores = list(strategy_rankings.values())
411 | score_std = (
412 | sum((s - sum(all_scores) / len(all_scores)) ** 2 for s in all_scores) ** 0.5
413 | )
414 | score_spread = (
415 | score_std / (sum(all_scores) / len(all_scores)) if all_scores else 0
416 | )
417 |
418 | # Combine factors
419 | fitness_confidence = avg_selected_fitness # 0-1
420 | spread_confidence = min(score_spread, 1.0) # Normalize spread
421 |
422 | # Weight by regime confidence
423 | total_confidence = (
424 | fitness_confidence * 0.5 + spread_confidence * 0.2 + regime_confidence * 0.3
425 | )
426 |
427 | return max(0.1, min(0.95, total_confidence))
428 |
429 | def get_strategy_compatibility_matrix(self) -> dict[str, dict[str, float]]:
430 | """Get compatibility matrix showing strategy fitness for each regime."""
431 | return self.REGIME_STRATEGY_FITNESS.copy()
432 |
433 | def explain_strategy_selection(
434 | self, regime: str, strategy: str, market_conditions: dict[str, Any]
435 | ) -> str:
436 | """Explain why a specific strategy is suitable for given conditions."""
437 | base_fitness = self.REGIME_STRATEGY_FITNESS.get(regime, {}).get(strategy, 0.5)
438 |
439 | explanations = {
440 | "sma_cross": {
441 | "trending": "SMA crossovers excel in trending markets by catching trend changes early.",
442 | "ranging": "SMA crossovers produce many false signals in ranging markets.",
443 | },
444 | "rsi": {
445 | "ranging": "RSI is ideal for ranging markets, buying oversold and selling overbought levels.",
446 | "trending": "RSI can remain overbought/oversold for extended periods in strong trends.",
447 | },
448 | "breakout": {
449 | "volatile": "Breakout strategies capitalize on high volatility and strong price moves.",
450 | "ranging": "Breakout strategies struggle in ranging markets with frequent false breakouts.",
451 | },
452 | }
453 |
454 | specific_explanation = explanations.get(strategy, {}).get(regime, "")
455 |
456 | return f"Strategy '{strategy}' has {base_fitness:.1%} fitness for '{regime}' markets. {specific_explanation}"
457 |
```
--------------------------------------------------------------------------------
/scripts/seed_sp500.py:
--------------------------------------------------------------------------------
```python
1 | #!/usr/bin/env python3
2 | """
3 | S&P 500 database seeding script for MaverickMCP.
4 |
5 | This script populates the database with all S&P 500 stocks, including
6 | company information, sector data, and comprehensive stock details.
7 | """
8 |
9 | import logging
10 | import os
11 | import sys
12 | import time
13 | from pathlib import Path
14 |
15 | # Add the project root to the Python path
16 | project_root = Path(__file__).parent.parent
17 | sys.path.insert(0, str(project_root))
18 |
19 | # noqa: E402 - imports must come after sys.path modification
20 | import pandas as pd # noqa: E402
21 | import yfinance as yf # noqa: E402
22 | from sqlalchemy import create_engine, text # noqa: E402
23 | from sqlalchemy.orm import sessionmaker # noqa: E402
24 |
25 | from maverick_mcp.data.models import ( # noqa: E402
26 | MaverickBearStocks,
27 | MaverickStocks,
28 | PriceCache,
29 | Stock,
30 | SupplyDemandBreakoutStocks,
31 | TechnicalCache,
32 | bulk_insert_screening_data,
33 | )
34 |
35 | # Set up logging
36 | logging.basicConfig(
37 | level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
38 | )
39 | logger = logging.getLogger("maverick_mcp.seed_sp500")
40 |
41 |
42 | def get_database_url() -> str:
43 | """Get the database URL from environment or settings."""
44 | return os.getenv("DATABASE_URL") or "sqlite:///maverick_mcp.db"
45 |
46 |
47 | def fetch_sp500_list() -> pd.DataFrame:
48 | """Fetch the current S&P 500 stock list from Wikipedia."""
49 | logger.info("Fetching S&P 500 stock list from Wikipedia...")
50 |
51 | try:
52 | # Read S&P 500 list from Wikipedia
53 | tables = pd.read_html(
54 | "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
55 | )
56 | sp500_df = tables[0] # First table contains the stock list
57 |
58 | # Clean up column names
59 | sp500_df.columns = [
60 | "symbol",
61 | "company",
62 | "gics_sector",
63 | "gics_sub_industry",
64 | "headquarters",
65 | "date_added",
66 | "cik",
67 | "founded",
68 | ]
69 |
70 | # Clean symbol column (remove any extra characters)
71 | sp500_df["symbol"] = sp500_df["symbol"].str.replace(".", "-", regex=False)
72 |
73 | logger.info(f"Fetched {len(sp500_df)} S&P 500 companies")
74 | return sp500_df[
75 | ["symbol", "company", "gics_sector", "gics_sub_industry"]
76 | ].copy()
77 |
78 | except Exception as e:
79 | logger.error(f"Failed to fetch S&P 500 list from Wikipedia: {e}")
80 | logger.info("Falling back to manually curated S&P 500 list...")
81 |
82 | # Fallback to a curated list of major S&P 500 stocks
83 | fallback_stocks = [
84 | (
85 | "AAPL",
86 | "Apple Inc.",
87 | "Information Technology",
88 | "Technology Hardware, Storage & Peripherals",
89 | ),
90 | ("MSFT", "Microsoft Corporation", "Information Technology", "Software"),
91 | (
92 | "GOOGL",
93 | "Alphabet Inc.",
94 | "Communication Services",
95 | "Interactive Media & Services",
96 | ),
97 | (
98 | "AMZN",
99 | "Amazon.com Inc.",
100 | "Consumer Discretionary",
101 | "Internet & Direct Marketing Retail",
102 | ),
103 | ("TSLA", "Tesla Inc.", "Consumer Discretionary", "Automobiles"),
104 | (
105 | "NVDA",
106 | "NVIDIA Corporation",
107 | "Information Technology",
108 | "Semiconductors & Semiconductor Equipment",
109 | ),
110 | (
111 | "META",
112 | "Meta Platforms Inc.",
113 | "Communication Services",
114 | "Interactive Media & Services",
115 | ),
116 | ("BRK-B", "Berkshire Hathaway Inc.", "Financials", "Multi-Sector Holdings"),
117 | ("JNJ", "Johnson & Johnson", "Health Care", "Pharmaceuticals"),
118 | (
119 | "V",
120 | "Visa Inc.",
121 | "Information Technology",
122 | "Data Processing & Outsourced Services",
123 | ),
124 | # Add more major S&P 500 companies
125 | ("JPM", "JPMorgan Chase & Co.", "Financials", "Banks"),
126 | ("WMT", "Walmart Inc.", "Consumer Staples", "Food & Staples Retailing"),
127 | ("PG", "Procter & Gamble Co.", "Consumer Staples", "Household Products"),
128 | (
129 | "UNH",
130 | "UnitedHealth Group Inc.",
131 | "Health Care",
132 | "Health Care Providers & Services",
133 | ),
134 | (
135 | "MA",
136 | "Mastercard Inc.",
137 | "Information Technology",
138 | "Data Processing & Outsourced Services",
139 | ),
140 | ("HD", "Home Depot Inc.", "Consumer Discretionary", "Specialty Retail"),
141 | ("BAC", "Bank of America Corp.", "Financials", "Banks"),
142 | ("PFE", "Pfizer Inc.", "Health Care", "Pharmaceuticals"),
143 | ("KO", "Coca-Cola Co.", "Consumer Staples", "Beverages"),
144 | ("ABBV", "AbbVie Inc.", "Health Care", "Pharmaceuticals"),
145 | ]
146 |
147 | fallback_df = pd.DataFrame(
148 | fallback_stocks,
149 | columns=["symbol", "company", "gics_sector", "gics_sub_industry"],
150 | )
151 | logger.info(
152 | f"Using fallback list with {len(fallback_df)} major S&P 500 companies"
153 | )
154 | return fallback_df
155 |
156 |
157 | def enrich_stock_data(symbol: str) -> dict:
158 | """Enrich stock data with additional information from yfinance."""
159 | try:
160 | ticker = yf.Ticker(symbol)
161 | info = ticker.info
162 |
163 | # Extract relevant information
164 | enriched_data = {
165 | "market_cap": info.get("marketCap"),
166 | "shares_outstanding": info.get("sharesOutstanding"),
167 | "description": info.get("longBusinessSummary", ""),
168 | "country": info.get("country", "US"),
169 | "currency": info.get("currency", "USD"),
170 | "exchange": info.get("exchange", "NASDAQ"),
171 | "industry": info.get("industry", ""),
172 | "sector": info.get("sector", ""),
173 | }
174 |
175 | # Clean up description (limit length)
176 | if enriched_data["description"] and len(enriched_data["description"]) > 500:
177 | enriched_data["description"] = enriched_data["description"][:500] + "..."
178 |
179 | return enriched_data
180 |
181 | except Exception as e:
182 | logger.warning(f"Failed to enrich data for {symbol}: {e}")
183 | return {}
184 |
185 |
186 | def create_sp500_stocks(session, sp500_df: pd.DataFrame) -> dict[str, Stock]:
187 | """Create S&P 500 stock records with comprehensive data."""
188 | logger.info(f"Creating {len(sp500_df)} S&P 500 stocks...")
189 |
190 | created_stocks = {}
191 | batch_size = 10
192 |
193 | for i, row in sp500_df.iterrows():
194 | symbol = row["symbol"]
195 | company = row["company"]
196 | gics_sector = row["gics_sector"]
197 | gics_sub_industry = row["gics_sub_industry"]
198 |
199 | try:
200 | logger.info(f"Processing {symbol} ({i + 1}/{len(sp500_df)})...")
201 |
202 | # Rate limiting - pause every batch to be nice to APIs
203 | if i > 0 and i % batch_size == 0:
204 | logger.info(f"Processed {i} stocks, pausing for 2 seconds...")
205 | time.sleep(2)
206 |
207 | # Enrich with additional data from yfinance
208 | enriched_data = enrich_stock_data(symbol)
209 |
210 | # Create stock record
211 | stock = Stock.get_or_create(
212 | session,
213 | ticker_symbol=symbol,
214 | company_name=company,
215 | sector=enriched_data.get("sector") or gics_sector or "Unknown",
216 | industry=enriched_data.get("industry")
217 | or gics_sub_industry
218 | or "Unknown",
219 | description=enriched_data.get("description")
220 | or f"{company} - S&P 500 component",
221 | exchange=enriched_data.get("exchange", "NASDAQ"),
222 | country=enriched_data.get("country", "US"),
223 | currency=enriched_data.get("currency", "USD"),
224 | market_cap=enriched_data.get("market_cap"),
225 | shares_outstanding=enriched_data.get("shares_outstanding"),
226 | is_active=True,
227 | )
228 |
229 | created_stocks[symbol] = stock
230 | logger.info(f"✓ Created {symbol}: {company}")
231 |
232 | except Exception as e:
233 | logger.error(f"✗ Error creating stock {symbol}: {e}")
234 | continue
235 |
236 | session.commit()
237 | logger.info(f"Successfully created {len(created_stocks)} S&P 500 stocks")
238 | return created_stocks
239 |
240 |
241 | def create_sample_screening_for_sp500(session, stocks: dict[str, Stock]) -> None:
242 | """Create sample screening results for S&P 500 stocks."""
243 | logger.info("Creating sample screening results for S&P 500 stocks...")
244 |
245 | # Generate screening data based on stock symbols
246 | screening_data = []
247 | stock_items = list(stocks.items())
248 |
249 | for _i, (ticker, _stock) in enumerate(stock_items):
250 | # Use hash of ticker for consistent "random" values
251 | ticker_hash = hash(ticker)
252 |
253 | # Generate realistic screening metrics
254 | base_price = 50 + (ticker_hash % 200) # Price between 50-250
255 | momentum_score = 30 + (ticker_hash % 70) # Score 30-100
256 |
257 | data = {
258 | "ticker": ticker,
259 | "close": round(base_price + (ticker_hash % 50), 2),
260 | "volume": 500000 + (ticker_hash % 10000000), # 0.5M - 10.5M volume
261 | "momentum_score": round(momentum_score, 2),
262 | "combined_score": min(100, momentum_score + (ticker_hash % 20)),
263 | "ema_21": round(base_price * 0.98, 2),
264 | "sma_50": round(base_price * 0.96, 2),
265 | "sma_150": round(base_price * 0.94, 2),
266 | "sma_200": round(base_price * 0.92, 2),
267 | "adr_pct": round(1.5 + (ticker_hash % 6), 2), # ADR 1.5-7.5%
268 | "atr": round(2 + (ticker_hash % 8), 2),
269 | "pattern_type": ["Breakout", "Continuation", "Reversal", "Base"][
270 | ticker_hash % 4
271 | ],
272 | "squeeze_status": ["No Squeeze", "Low", "Mid", "High"][ticker_hash % 4],
273 | "consolidation_status": ["Base", "Flag", "Pennant", "Triangle"][
274 | ticker_hash % 4
275 | ],
276 | "entry_signal": ["Buy", "Hold", "Watch", "Caution"][ticker_hash % 4],
277 | "compression_score": ticker_hash % 10,
278 | "pattern_detected": 1 if ticker_hash % 3 == 0 else 0,
279 | }
280 | screening_data.append(data)
281 |
282 | # Sort by combined score for different screening types
283 | total_stocks = len(screening_data)
284 |
285 | # Top 30% for Maverick (bullish momentum)
286 | maverick_count = max(10, int(total_stocks * 0.3)) # At least 10 stocks
287 | maverick_data = sorted(
288 | screening_data, key=lambda x: x["combined_score"], reverse=True
289 | )[:maverick_count]
290 | maverick_count = bulk_insert_screening_data(session, MaverickStocks, maverick_data)
291 | logger.info(f"Created {maverick_count} Maverick screening results")
292 |
293 | # Bottom 20% for Bear stocks (weak momentum)
294 | bear_count = max(5, int(total_stocks * 0.2)) # At least 5 stocks
295 | bear_data = sorted(screening_data, key=lambda x: x["combined_score"])[:bear_count]
296 |
297 | # Add bear-specific fields
298 | for data in bear_data:
299 | data["score"] = 100 - data["combined_score"] # Invert score
300 | data["rsi_14"] = 70 + (hash(data["ticker"]) % 25) # Overbought RSI
301 | data["macd"] = -0.1 - (hash(data["ticker"]) % 5) / 20 # Negative MACD
302 | data["macd_signal"] = -0.05 - (hash(data["ticker"]) % 3) / 30
303 | data["macd_histogram"] = data["macd"] - data["macd_signal"]
304 | data["dist_days_20"] = hash(data["ticker"]) % 20
305 | data["atr_contraction"] = hash(data["ticker"]) % 2 == 0
306 | data["big_down_vol"] = hash(data["ticker"]) % 4 == 0
307 |
308 | bear_inserted = bulk_insert_screening_data(session, MaverickBearStocks, bear_data)
309 | logger.info(f"Created {bear_inserted} Bear screening results")
310 |
311 | # Top 25% for Supply/Demand breakouts
312 | breakout_count = max(8, int(total_stocks * 0.25)) # At least 8 stocks
313 | breakout_data = sorted(
314 | screening_data, key=lambda x: x["momentum_score"], reverse=True
315 | )[:breakout_count]
316 |
317 | # Add supply/demand specific fields
318 | for data in breakout_data:
319 | data["accumulation_rating"] = 2 + (hash(data["ticker"]) % 8) # 2-9
320 | data["distribution_rating"] = 10 - data["accumulation_rating"]
321 | data["breakout_strength"] = 3 + (hash(data["ticker"]) % 7) # 3-9
322 | data["avg_volume_30d"] = data["volume"] * 1.3 # 30% above current
323 |
324 | breakout_inserted = bulk_insert_screening_data(
325 | session, SupplyDemandBreakoutStocks, breakout_data
326 | )
327 | logger.info(f"Created {breakout_inserted} Supply/Demand breakout results")
328 |
329 |
330 | def verify_sp500_data(session) -> None:
331 | """Verify that S&P 500 data was seeded correctly."""
332 | logger.info("Verifying S&P 500 seeded data...")
333 |
334 | # Count records in each table
335 | stock_count = session.query(Stock).count()
336 | price_count = session.query(PriceCache).count()
337 | maverick_count = session.query(MaverickStocks).count()
338 | bear_count = session.query(MaverickBearStocks).count()
339 | supply_demand_count = session.query(SupplyDemandBreakoutStocks).count()
340 | technical_count = session.query(TechnicalCache).count()
341 |
342 | logger.info("=== S&P 500 Data Seeding Summary ===")
343 | logger.info(f"S&P 500 Stocks: {stock_count}")
344 | logger.info(f"Price records: {price_count}")
345 | logger.info(f"Maverick screening: {maverick_count}")
346 | logger.info(f"Bear screening: {bear_count}")
347 | logger.info(f"Supply/Demand screening: {supply_demand_count}")
348 | logger.info(f"Technical indicators: {technical_count}")
349 | logger.info("===================================")
350 |
351 | # Show top stocks by sector
352 | logger.info("\n📊 S&P 500 Stocks by Sector:")
353 | sector_counts = session.execute(
354 | text("""
355 | SELECT sector, COUNT(*) as count
356 | FROM mcp_stocks
357 | WHERE sector IS NOT NULL
358 | GROUP BY sector
359 | ORDER BY count DESC
360 | LIMIT 10
361 | """)
362 | ).fetchall()
363 |
364 | for sector, count in sector_counts:
365 | logger.info(f" {sector}: {count} stocks")
366 |
367 | # Test screening queries
368 | if maverick_count > 0:
369 | top_maverick = (
370 | session.query(MaverickStocks)
371 | .order_by(MaverickStocks.combined_score.desc())
372 | .first()
373 | )
374 | if top_maverick and top_maverick.stock:
375 | logger.info(
376 | f"\n🚀 Top Maverick (Bullish): {top_maverick.stock.ticker_symbol} (Score: {top_maverick.combined_score})"
377 | )
378 |
379 | if bear_count > 0:
380 | top_bear = (
381 | session.query(MaverickBearStocks)
382 | .order_by(MaverickBearStocks.score.desc())
383 | .first()
384 | )
385 | if top_bear and top_bear.stock:
386 | logger.info(
387 | f"🐻 Top Bear: {top_bear.stock.ticker_symbol} (Score: {top_bear.score})"
388 | )
389 |
390 | if supply_demand_count > 0:
391 | top_breakout = (
392 | session.query(SupplyDemandBreakoutStocks)
393 | .order_by(SupplyDemandBreakoutStocks.breakout_strength.desc())
394 | .first()
395 | )
396 | if top_breakout and top_breakout.stock:
397 | logger.info(
398 | f"📈 Top Breakout: {top_breakout.stock.ticker_symbol} (Strength: {top_breakout.breakout_strength})"
399 | )
400 |
401 |
402 | def main():
403 | """Main S&P 500 seeding function."""
404 | logger.info("🚀 Starting S&P 500 database seeding for MaverickMCP...")
405 |
406 | # Set up database connection
407 | database_url = get_database_url()
408 | logger.info(f"Using database: {database_url}")
409 |
410 | engine = create_engine(database_url, echo=False)
411 | SessionLocal = sessionmaker(bind=engine)
412 |
413 | with SessionLocal() as session:
414 | try:
415 | # Fetch S&P 500 stock list
416 | sp500_df = fetch_sp500_list()
417 |
418 | if sp500_df.empty:
419 | logger.error("No S&P 500 stocks found. Exiting.")
420 | return False
421 |
422 | # Create S&P 500 stocks with comprehensive data
423 | stocks = create_sp500_stocks(session, sp500_df)
424 |
425 | if not stocks:
426 | logger.error("No S&P 500 stocks created. Exiting.")
427 | return False
428 |
429 | # Create screening results for S&P 500 stocks
430 | create_sample_screening_for_sp500(session, stocks)
431 |
432 | # Verify data
433 | verify_sp500_data(session)
434 |
435 | logger.info("🎉 S&P 500 database seeding completed successfully!")
436 | logger.info(f"📈 Added {len(stocks)} S&P 500 companies to the database")
437 | logger.info("\n🔧 Next steps:")
438 | logger.info("1. Run 'make dev' to start the MCP server")
439 | logger.info("2. Connect with Claude Desktop using mcp-remote")
440 | logger.info("3. Test with: 'Show me top S&P 500 momentum stocks'")
441 |
442 | return True
443 |
444 | except Exception as e:
445 | logger.error(f"S&P 500 database seeding failed: {e}")
446 | session.rollback()
447 | raise
448 |
449 |
450 | if __name__ == "__main__":
451 | try:
452 | success = main()
453 | if not success:
454 | sys.exit(1)
455 | except KeyboardInterrupt:
456 | logger.info("\n⏹️ Seeding interrupted by user")
457 | sys.exit(1)
458 | except Exception as e:
459 | logger.error(f"❌ Fatal error: {e}")
460 | sys.exit(1)
461 |
```
--------------------------------------------------------------------------------
/maverick_mcp/utils/monitoring_middleware.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Monitoring middleware for FastMCP and FastAPI applications.
3 |
4 | This module provides comprehensive monitoring middleware that automatically:
5 | - Tracks request metrics (count, duration, size)
6 | - Creates distributed traces for all requests
7 | - Monitors database and cache operations
8 | - Tracks business metrics and user behavior
9 | - Integrates with Prometheus and OpenTelemetry
10 | """
11 |
12 | import time
13 | from collections.abc import Callable
14 | from typing import Any
15 |
16 | from fastapi import Request, Response
17 | from starlette.middleware.base import BaseHTTPMiddleware
18 |
19 | from maverick_mcp.utils.logging import get_logger, user_id_var
20 | from maverick_mcp.utils.monitoring import (
21 | active_connections,
22 | concurrent_requests,
23 | request_counter,
24 | request_duration,
25 | request_size_bytes,
26 | response_size_bytes,
27 | track_authentication,
28 | track_rate_limit_hit,
29 | track_security_violation,
30 | update_performance_metrics,
31 | )
32 | from maverick_mcp.utils.tracing import get_tracing_service, trace_operation
33 |
34 | logger = get_logger(__name__)
35 |
36 |
37 | class MonitoringMiddleware(BaseHTTPMiddleware):
38 | """
39 | Comprehensive monitoring middleware for FastAPI applications.
40 |
41 | Automatically tracks:
42 | - Request/response metrics
43 | - Distributed tracing
44 | - Performance monitoring
45 | - Security events
46 | - Business metrics
47 | """
48 |
49 | def __init__(self, app, enable_detailed_logging: bool = True):
50 | super().__init__(app)
51 | self.enable_detailed_logging = enable_detailed_logging
52 | self.tracing = get_tracing_service()
53 |
54 | async def dispatch(self, request: Request, call_next: Callable) -> Response:
55 | """Process request with comprehensive monitoring."""
56 | # Start timing
57 | start_time = time.time()
58 |
59 | # Track active connections
60 | active_connections.inc()
61 | concurrent_requests.inc()
62 |
63 | # Extract request information
64 | method = request.method
65 | path = request.url.path
66 | endpoint = self._normalize_endpoint(path)
67 | user_agent = request.headers.get("user-agent", "unknown")
68 |
69 | # Calculate request size
70 | content_length = request.headers.get("content-length")
71 | req_size = int(content_length) if content_length else 0
72 |
73 | # Extract user information for monitoring
74 | user_id = self._extract_user_id(request)
75 | user_type = self._determine_user_type(request, user_id)
76 |
77 | # Set context variables for logging
78 | if user_id:
79 | user_id_var.set(str(user_id))
80 |
81 | response = None
82 | status_code = 500
83 | error_type = None
84 |
85 | # Create tracing span for the entire request
86 | with trace_operation(
87 | f"{method} {endpoint}",
88 | attributes={
89 | "http.method": method,
90 | "http.route": endpoint,
91 | "http.user_agent": user_agent[:100], # Truncate long user agents
92 | "user.id": str(user_id) if user_id else "anonymous",
93 | "user.type": user_type,
94 | "http.request_size": req_size,
95 | },
96 | ) as span:
97 | try:
98 | # Process the request
99 | response = await call_next(request)
100 | status_code = response.status_code
101 |
102 | # Track successful request
103 | if span:
104 | span.set_attribute("http.status_code", status_code)
105 | span.set_attribute(
106 | "http.response_size", self._get_response_size(response)
107 | )
108 |
109 | # Track authentication events
110 | if self._is_auth_endpoint(endpoint):
111 | auth_status = "success" if 200 <= status_code < 300 else "failure"
112 | track_authentication(
113 | method="bearer_token",
114 | status=auth_status,
115 | user_agent=user_agent[:50],
116 | )
117 |
118 | # Track rate limiting
119 | if status_code == 429:
120 | track_rate_limit_hit(
121 | user_id=str(user_id) if user_id else "anonymous",
122 | endpoint=endpoint,
123 | limit_type="request_rate",
124 | )
125 |
126 | except Exception as e:
127 | error_type = type(e).__name__
128 | status_code = 500
129 |
130 | # Record exception in trace
131 | if span:
132 | span.record_exception(e)
133 | span.set_attribute("error", True)
134 | span.set_attribute("error.type", error_type)
135 |
136 | # Track security violations for certain errors
137 | if self._is_security_error(e):
138 | track_security_violation(
139 | violation_type=error_type,
140 | severity="high" if status_code >= 400 else "medium",
141 | )
142 |
143 | # Re-raise the exception
144 | raise
145 |
146 | finally:
147 | # Calculate duration
148 | duration = time.time() - start_time
149 |
150 | # Determine final status for metrics
151 | final_status = "success" if 200 <= status_code < 400 else "error"
152 |
153 | # Track request metrics
154 | request_counter.labels(
155 | method=method,
156 | endpoint=endpoint,
157 | status=final_status,
158 | user_type=user_type,
159 | ).inc()
160 |
161 | request_duration.labels(
162 | method=method, endpoint=endpoint, user_type=user_type
163 | ).observe(duration)
164 |
165 | # Track request/response sizes
166 | if req_size > 0:
167 | request_size_bytes.labels(method=method, endpoint=endpoint).observe(
168 | req_size
169 | )
170 |
171 | if response:
172 | resp_size = self._get_response_size(response)
173 | if resp_size > 0:
174 | response_size_bytes.labels(
175 | method=method, endpoint=endpoint, status=str(status_code)
176 | ).observe(resp_size)
177 |
178 | # Update performance metrics periodically
179 | if int(time.time()) % 30 == 0: # Every 30 seconds
180 | try:
181 | update_performance_metrics()
182 | except Exception as e:
183 | logger.warning(f"Failed to update performance metrics: {e}")
184 |
185 | # Log detailed request information
186 | if self.enable_detailed_logging:
187 | self._log_request_details(
188 | method, endpoint, status_code, duration, user_id, error_type
189 | )
190 |
191 | # Update connection counters
192 | active_connections.dec()
193 | concurrent_requests.dec()
194 |
195 | return response
196 |
197 | def _normalize_endpoint(self, path: str) -> str:
198 | """Normalize endpoint path for metrics (replace IDs with placeholders)."""
199 | # Replace UUIDs and IDs in paths
200 | import re
201 |
202 | # Replace UUID patterns
203 | path = re.sub(r"/[a-f0-9-]{36}", "/{uuid}", path)
204 |
205 | # Replace numeric IDs
206 | path = re.sub(r"/\d+", "/{id}", path)
207 |
208 | # Replace API keys or tokens
209 | path = re.sub(r"/[a-zA-Z0-9]{20,}", "/{token}", path)
210 |
211 | return path
212 |
213 | def _extract_user_id(self, request: Request) -> str | None:
214 | """Extract user ID from request (from JWT, session, etc.)."""
215 | # Check Authorization header
216 | auth_header = request.headers.get("authorization")
217 | if auth_header and auth_header.startswith("Bearer "):
218 | try:
219 | # In a real implementation, you'd decode the JWT
220 | # For now, we'll check if there's a user context
221 | if hasattr(request.state, "user_id"):
222 | return request.state.user_id
223 | except Exception:
224 | pass
225 |
226 | # Check for user ID in path parameters
227 | if hasattr(request, "path_params") and "user_id" in request.path_params:
228 | return request.path_params["user_id"]
229 |
230 | return None
231 |
232 | def _determine_user_type(self, request: Request, user_id: str | None) -> str:
233 | """Determine user type for metrics."""
234 | if not user_id:
235 | return "anonymous"
236 |
237 | # Check if it's an admin user (you'd implement your own logic)
238 | if hasattr(request.state, "user_role"):
239 | return request.state.user_role
240 |
241 | # Check for API key usage
242 | if request.headers.get("x-api-key"):
243 | return "api_user"
244 |
245 | return "authenticated"
246 |
247 | def _is_auth_endpoint(self, endpoint: str) -> bool:
248 | """Check if endpoint is authentication-related."""
249 | auth_endpoints = ["/login", "/auth", "/token", "/signup", "/register"]
250 | return any(auth_ep in endpoint for auth_ep in auth_endpoints)
251 |
252 | def _is_security_error(self, exception: Exception) -> bool:
253 | """Check if exception indicates a security issue."""
254 | security_errors = [
255 | "PermissionError",
256 | "Unauthorized",
257 | "Forbidden",
258 | "ValidationError",
259 | "SecurityError",
260 | ]
261 | return any(error in str(type(exception)) for error in security_errors)
262 |
263 | def _get_response_size(self, response: Response) -> int:
264 | """Calculate response size in bytes."""
265 | content_length = response.headers.get("content-length")
266 | if content_length:
267 | return int(content_length)
268 |
269 | # Estimate size if content-length is not set
270 | if hasattr(response, "body") and response.body:
271 | return len(response.body)
272 |
273 | return 0
274 |
275 | def _log_request_details(
276 | self,
277 | method: str,
278 | endpoint: str,
279 | status_code: int,
280 | duration: float,
281 | user_id: str | None,
282 | error_type: str | None,
283 | ):
284 | """Log detailed request information."""
285 | log_data = {
286 | "http_method": method,
287 | "endpoint": endpoint,
288 | "status_code": status_code,
289 | "duration_ms": int(duration * 1000),
290 | "user_id": str(user_id) if user_id else None,
291 | }
292 |
293 | if error_type:
294 | log_data["error_type"] = error_type
295 |
296 | if status_code >= 400:
297 | logger.warning(f"HTTP {status_code}: {method} {endpoint}", extra=log_data)
298 | else:
299 | logger.info(f"HTTP {status_code}: {method} {endpoint}", extra=log_data)
300 |
301 |
302 | class MCPToolMonitoringWrapper:
303 | """
304 | Wrapper for MCP tools to add monitoring capabilities.
305 |
306 | This class wraps MCP tool execution to automatically:
307 | - Track tool usage metrics
308 | - Create distributed traces
309 | - Monitor performance
310 | """
311 |
312 | def __init__(self, enable_tracing: bool = True):
313 | self.enable_tracing = enable_tracing
314 | self.tracing = get_tracing_service()
315 |
316 | def monitor_tool(self, tool_func: Callable) -> Callable:
317 | """
318 | Decorator to add monitoring to MCP tools.
319 |
320 | Args:
321 | tool_func: The MCP tool function to monitor
322 |
323 | Returns:
324 | Wrapped function with monitoring
325 | """
326 | from functools import wraps
327 |
328 | @wraps(tool_func)
329 | async def wrapper(*args, **kwargs):
330 | tool_name = tool_func.__name__
331 | start_time = time.time()
332 |
333 | # Extract user context from args
334 | user_id = None
335 | for arg in args:
336 | if hasattr(arg, "user_id"):
337 | user_id = arg.user_id
338 | break
339 | # Check if it's an MCP context
340 | if hasattr(arg, "__class__") and "Context" in arg.__class__.__name__:
341 | # Extract user from context if available
342 | if hasattr(arg, "user_id"):
343 | user_id = arg.user_id
344 |
345 | # Set context for logging
346 | if user_id:
347 | user_id_var.set(str(user_id))
348 |
349 | # Create tracing span
350 | with trace_operation(
351 | f"tool.{tool_name}",
352 | attributes={
353 | "tool.name": tool_name,
354 | "user.id": str(user_id) if user_id else "anonymous",
355 | "tool.args_count": len(args),
356 | "tool.kwargs_count": len(kwargs),
357 | },
358 | ) as span:
359 | try:
360 | # Execute the tool
361 | result = await tool_func(*args, **kwargs)
362 |
363 | # Calculate execution time
364 | duration = time.time() - start_time
365 |
366 | # Track successful execution
367 | from maverick_mcp.utils.monitoring import track_tool_usage
368 |
369 | track_tool_usage(
370 | tool_name=tool_name,
371 | user_id=str(user_id) if user_id else "anonymous",
372 | duration=duration,
373 | status="success",
374 | complexity=self._determine_complexity(tool_name, kwargs),
375 | )
376 |
377 | # Add attributes to span
378 | if span:
379 | span.set_attribute("tool.duration_seconds", duration)
380 | span.set_attribute("tool.success", True)
381 | span.set_attribute("tool.result_size", len(str(result)))
382 |
383 | # Add usage information to result if it's a dict
384 | if isinstance(result, dict):
385 | result["_monitoring"] = {
386 | "execution_time_ms": int(duration * 1000),
387 | "tool_name": tool_name,
388 | "timestamp": time.time(),
389 | }
390 |
391 | return result
392 |
393 | except Exception as e:
394 | # Calculate execution time
395 | duration = time.time() - start_time
396 | error_type = type(e).__name__
397 |
398 | # Track failed execution
399 | from maverick_mcp.utils.monitoring import track_tool_error
400 |
401 | track_tool_error(
402 | tool_name=tool_name,
403 | error_type=error_type,
404 | complexity=self._determine_complexity(tool_name, kwargs),
405 | )
406 |
407 | # Add error attributes to span
408 | if span:
409 | span.set_attribute("tool.duration_seconds", duration)
410 | span.set_attribute("tool.success", False)
411 | span.set_attribute("error.type", error_type)
412 | span.record_exception(e)
413 |
414 | logger.error(
415 | f"Tool execution failed: {tool_name}",
416 | extra={
417 | "tool_name": tool_name,
418 | "user_id": str(user_id) if user_id else None,
419 | "duration_ms": int(duration * 1000),
420 | "error_type": error_type,
421 | },
422 | exc_info=True,
423 | )
424 |
425 | # Re-raise the exception
426 | raise
427 |
428 | return wrapper
429 |
430 | def _determine_complexity(self, tool_name: str, kwargs: dict[str, Any]) -> str:
431 | """Determine tool complexity based on parameters."""
432 | # Simple heuristics for complexity
433 | if "limit" in kwargs:
434 | limit = kwargs.get("limit", 0)
435 | if limit > 100:
436 | return "high"
437 | elif limit > 50:
438 | return "medium"
439 |
440 | if "symbols" in kwargs:
441 | symbols = kwargs.get("symbols", [])
442 | if isinstance(symbols, list) and len(symbols) > 10:
443 | return "high"
444 | elif isinstance(symbols, list) and len(symbols) > 5:
445 | return "medium"
446 |
447 | # Check for complex analysis tools
448 | complex_tools = [
449 | "get_portfolio_optimization",
450 | "get_market_analysis",
451 | "screen_stocks",
452 | ]
453 | if any(complex_tool in tool_name for complex_tool in complex_tools):
454 | return "high"
455 |
456 | return "standard"
457 |
458 |
459 | def create_monitoring_middleware(
460 | enable_detailed_logging: bool = True,
461 | ) -> MonitoringMiddleware:
462 | """Create a monitoring middleware instance."""
463 | return MonitoringMiddleware(enable_detailed_logging=enable_detailed_logging)
464 |
465 |
466 | def create_tool_monitor(enable_tracing: bool = True) -> MCPToolMonitoringWrapper:
467 | """Create a tool monitoring wrapper instance."""
468 | return MCPToolMonitoringWrapper(enable_tracing=enable_tracing)
469 |
470 |
471 | # Global instances
472 | _monitoring_middleware: MonitoringMiddleware | None = None
473 | _tool_monitor: MCPToolMonitoringWrapper | None = None
474 |
475 |
476 | def get_monitoring_middleware() -> MonitoringMiddleware:
477 | """Get or create the global monitoring middleware."""
478 | global _monitoring_middleware
479 | if _monitoring_middleware is None:
480 | _monitoring_middleware = create_monitoring_middleware()
481 | return _monitoring_middleware
482 |
483 |
484 | def get_tool_monitor() -> MCPToolMonitoringWrapper:
485 | """Get or create the global tool monitor."""
486 | global _tool_monitor
487 | if _tool_monitor is None:
488 | _tool_monitor = create_tool_monitor()
489 | return _tool_monitor
490 |
```
--------------------------------------------------------------------------------
/examples/llm_speed_demo.py:
--------------------------------------------------------------------------------
```python
1 | #!/usr/bin/env python3
2 | """
3 | Focused LLM Speed Optimization Demonstration
4 |
5 | This script demonstrates the core LLM optimization capabilities that provide
6 | 2-3x speed improvements, focusing on areas we can control directly.
7 |
8 | Demonstrates:
9 | - Adaptive model selection based on time constraints
10 | - Fast model execution (Gemini 2.5 Flash)
11 | - Token generation speed optimization
12 | - Progressive timeout management
13 | - Model performance comparison
14 | """
15 |
16 | import asyncio
17 | import os
18 | import sys
19 | import time
20 | from datetime import datetime
21 | from typing import Any
22 |
23 | # Add the project root to Python path
24 | sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
25 |
26 | from maverick_mcp.providers.openrouter_provider import OpenRouterProvider, TaskType
27 | from maverick_mcp.utils.llm_optimization import AdaptiveModelSelector
28 |
29 |
30 | class LLMSpeedDemonstrator:
31 | """Focused demonstration of LLM speed optimizations."""
32 |
33 | def __init__(self):
34 | """Initialize the demonstration."""
35 | api_key = os.getenv("OPENROUTER_API_KEY")
36 | if not api_key:
37 | raise ValueError(
38 | "OPENROUTER_API_KEY environment variable is required. "
39 | "Please set it with your OpenRouter API key."
40 | )
41 | self.openrouter_provider = OpenRouterProvider(api_key=api_key)
42 | self.model_selector = AdaptiveModelSelector(self.openrouter_provider)
43 |
44 | # Test scenarios focused on different urgency levels
45 | self.test_scenarios = [
46 | {
47 | "name": "Emergency Analysis (Critical Speed)",
48 | "prompt": "Analyze NVIDIA's latest earnings impact on AI market sentiment. 2-3 key points only.",
49 | "time_budget": 15.0,
50 | "task_type": TaskType.QUICK_ANSWER,
51 | "expected_speed": ">100 tok/s",
52 | },
53 | {
54 | "name": "Technical Analysis (Fast Response)",
55 | "prompt": "Provide technical analysis of Apple stock including RSI, MACD, and support levels. Be concise.",
56 | "time_budget": 30.0,
57 | "task_type": TaskType.TECHNICAL_ANALYSIS,
58 | "expected_speed": ">80 tok/s",
59 | },
60 | {
61 | "name": "Market Research (Moderate Speed)",
62 | "prompt": "Analyze Federal Reserve interest rate policy impact on technology sector. Include risk assessment.",
63 | "time_budget": 45.0,
64 | "task_type": TaskType.MARKET_ANALYSIS,
65 | "expected_speed": ">60 tok/s",
66 | },
67 | {
68 | "name": "Complex Synthesis (Quality Balance)",
69 | "prompt": "Synthesize renewable energy investment opportunities for 2025, considering policy changes, technology advances, and market trends.",
70 | "time_budget": 60.0,
71 | "task_type": TaskType.RESULT_SYNTHESIS,
72 | "expected_speed": ">40 tok/s",
73 | },
74 | ]
75 |
76 | def print_header(self, title: str):
77 | """Print formatted header."""
78 | print("\n" + "=" * 80)
79 | print(f" {title}")
80 | print("=" * 80)
81 |
82 | def print_subheader(self, title: str):
83 | """Print formatted subheader."""
84 | print(f"\n--- {title} ---")
85 |
86 | async def validate_openrouter_connection(self) -> bool:
87 | """Validate OpenRouter API is accessible."""
88 | self.print_header("🔧 API VALIDATION")
89 |
90 | try:
91 | test_llm = self.openrouter_provider.get_llm(TaskType.GENERAL)
92 | from langchain_core.messages import HumanMessage
93 |
94 | test_response = await asyncio.wait_for(
95 | test_llm.ainvoke([HumanMessage(content="test connection")]),
96 | timeout=10.0,
97 | )
98 | print("✅ OpenRouter API: Connected successfully")
99 | print(f" Response length: {len(test_response.content)} chars")
100 | return True
101 | except Exception as e:
102 | print(f"❌ OpenRouter API: Failed - {e}")
103 | return False
104 |
105 | async def demonstrate_model_selection(self):
106 | """Show intelligent model selection for different scenarios."""
107 | self.print_header("🧠 ADAPTIVE MODEL SELECTION")
108 |
109 | for scenario in self.test_scenarios:
110 | print(f"\n📋 Scenario: {scenario['name']}")
111 | print(f" Time Budget: {scenario['time_budget']}s")
112 | print(f" Task Type: {scenario['task_type'].value}")
113 | print(f" Expected Speed: {scenario['expected_speed']}")
114 |
115 | # Calculate task complexity
116 | complexity = self.model_selector.calculate_task_complexity(
117 | content=scenario["prompt"],
118 | task_type=scenario["task_type"],
119 | focus_areas=["analysis"],
120 | )
121 |
122 | # Get optimal model for time budget
123 | model_config = self.model_selector.select_model_for_time_budget(
124 | task_type=scenario["task_type"],
125 | time_remaining_seconds=scenario["time_budget"],
126 | complexity_score=complexity,
127 | content_size_tokens=len(scenario["prompt"]) // 4,
128 | )
129 |
130 | print(f" 📊 Complexity Score: {complexity:.2f}")
131 | print(f" 🎯 Selected Model: {model_config.model_id}")
132 | print(f" ⏱️ Max Timeout: {model_config.timeout_seconds}s")
133 | print(f" 🌡️ Temperature: {model_config.temperature}")
134 | print(f" 📝 Max Tokens: {model_config.max_tokens}")
135 |
136 | # Check if speed-optimized
137 | is_speed_model = any(
138 | x in model_config.model_id.lower()
139 | for x in ["flash", "haiku", "4o-mini", "deepseek"]
140 | )
141 | print(f" 🚀 Speed Optimized: {'✅' if is_speed_model else '❌'}")
142 |
143 | async def run_speed_benchmarks(self):
144 | """Run actual speed benchmarks for each scenario."""
145 | self.print_header("⚡ LIVE SPEED BENCHMARKS")
146 |
147 | results = []
148 | baseline_time = 60.0 # Historical baseline from timeout issues
149 |
150 | for i, scenario in enumerate(self.test_scenarios, 1):
151 | print(f"\n🔍 Benchmark {i}/{len(self.test_scenarios)}: {scenario['name']}")
152 | print(f" Query: {scenario['prompt'][:60]}...")
153 |
154 | try:
155 | # Get optimal model configuration
156 | complexity = self.model_selector.calculate_task_complexity(
157 | content=scenario["prompt"],
158 | task_type=scenario["task_type"],
159 | )
160 |
161 | model_config = self.model_selector.select_model_for_time_budget(
162 | task_type=scenario["task_type"],
163 | time_remaining_seconds=scenario["time_budget"],
164 | complexity_score=complexity,
165 | content_size_tokens=len(scenario["prompt"]) // 4,
166 | )
167 |
168 | # Execute with timing
169 | llm = self.openrouter_provider.get_llm(
170 | model_override=model_config.model_id,
171 | temperature=model_config.temperature,
172 | max_tokens=model_config.max_tokens,
173 | )
174 |
175 | start_time = time.time()
176 | from langchain_core.messages import HumanMessage
177 |
178 | response = await asyncio.wait_for(
179 | llm.ainvoke([HumanMessage(content=scenario["prompt"])]),
180 | timeout=model_config.timeout_seconds,
181 | )
182 | execution_time = time.time() - start_time
183 |
184 | # Calculate metrics
185 | response_length = len(response.content)
186 | estimated_tokens = response_length // 4
187 | tokens_per_second = (
188 | estimated_tokens / execution_time if execution_time > 0 else 0
189 | )
190 | speed_improvement = (
191 | baseline_time / execution_time if execution_time > 0 else 0
192 | )
193 |
194 | # Results
195 | result = {
196 | "scenario": scenario["name"],
197 | "model_used": model_config.model_id,
198 | "execution_time": execution_time,
199 | "time_budget": scenario["time_budget"],
200 | "budget_used_pct": (execution_time / scenario["time_budget"]) * 100,
201 | "tokens_per_second": tokens_per_second,
202 | "response_length": response_length,
203 | "speed_improvement": speed_improvement,
204 | "target_achieved": execution_time <= scenario["time_budget"],
205 | "response_preview": response.content[:150] + "..."
206 | if len(response.content) > 150
207 | else response.content,
208 | }
209 |
210 | results.append(result)
211 |
212 | # Print immediate results
213 | status_icon = "✅" if result["target_achieved"] else "⚠️"
214 | print(
215 | f" {status_icon} Completed: {execution_time:.2f}s ({result['budget_used_pct']:.1f}% of budget)"
216 | )
217 | print(f" 🎯 Model: {model_config.model_id}")
218 | print(f" 🚀 Speed: {tokens_per_second:.0f} tok/s")
219 | print(
220 | f" 📊 Improvement: {speed_improvement:.1f}x faster than baseline"
221 | )
222 | print(f" 💬 Preview: {result['response_preview']}")
223 |
224 | # Brief pause between tests
225 | await asyncio.sleep(1)
226 |
227 | except Exception as e:
228 | print(f" ❌ Failed: {str(e)}")
229 | results.append(
230 | {
231 | "scenario": scenario["name"],
232 | "error": str(e),
233 | "target_achieved": False,
234 | }
235 | )
236 |
237 | return results
238 |
239 | def analyze_benchmark_results(self, results: list[dict[str, Any]]):
240 | """Analyze and report benchmark results."""
241 | self.print_header("📊 SPEED OPTIMIZATION ANALYSIS")
242 |
243 | successful_tests = [r for r in results if not r.get("error")]
244 | failed_tests = [r for r in results if r.get("error")]
245 | targets_achieved = [r for r in successful_tests if r.get("target_achieved")]
246 |
247 | print("📈 Overall Performance:")
248 | print(f" Total Tests: {len(results)}")
249 | print(f" Successful: {len(successful_tests)}")
250 | print(f" Failed: {len(failed_tests)}")
251 | print(f" Targets Hit: {len(targets_achieved)}/{len(results)}")
252 | print(f" Success Rate: {(len(targets_achieved) / len(results) * 100):.1f}%")
253 |
254 | if successful_tests:
255 | # Speed metrics
256 | avg_execution_time = sum(
257 | r["execution_time"] for r in successful_tests
258 | ) / len(successful_tests)
259 | max_execution_time = max(r["execution_time"] for r in successful_tests)
260 | avg_tokens_per_second = sum(
261 | r["tokens_per_second"] for r in successful_tests
262 | ) / len(successful_tests)
263 | avg_speed_improvement = sum(
264 | r["speed_improvement"] for r in successful_tests
265 | ) / len(successful_tests)
266 |
267 | print("\n⚡ Speed Metrics:")
268 | print(f" Average Execution Time: {avg_execution_time:.2f}s")
269 | print(f" Maximum Execution Time: {max_execution_time:.2f}s")
270 | print(f" Average Token Generation: {avg_tokens_per_second:.0f} tok/s")
271 | print(f" Average Speed Improvement: {avg_speed_improvement:.1f}x")
272 |
273 | # Historical comparison
274 | historical_baseline = 60.0 # Average timeout failure time
275 | if max_execution_time > 0:
276 | overall_improvement = historical_baseline / max_execution_time
277 | print("\n🎯 Speed Validation:")
278 | print(
279 | f" Historical Baseline: {historical_baseline}s (timeout failures)"
280 | )
281 | print(f" Current Max Time: {max_execution_time:.2f}s")
282 | print(f" Overall Improvement: {overall_improvement:.1f}x")
283 |
284 | if overall_improvement >= 3.0:
285 | print(
286 | f" 🎉 EXCELLENT: {overall_improvement:.1f}x speed improvement!"
287 | )
288 | elif overall_improvement >= 2.0:
289 | print(
290 | f" ✅ SUCCESS: {overall_improvement:.1f}x speed improvement achieved!"
291 | )
292 | elif overall_improvement >= 1.5:
293 | print(
294 | f" 👍 GOOD: {overall_improvement:.1f}x improvement (target: 2x)"
295 | )
296 | else:
297 | print(
298 | f" ⚠️ NEEDS WORK: Only {overall_improvement:.1f}x improvement"
299 | )
300 |
301 | # Model performance breakdown
302 | self.print_subheader("🧠 MODEL PERFORMANCE BREAKDOWN")
303 | model_stats = {}
304 | for result in successful_tests:
305 | model = result["model_used"]
306 | if model not in model_stats:
307 | model_stats[model] = []
308 | model_stats[model].append(result)
309 |
310 | for model, model_results in model_stats.items():
311 | avg_speed = sum(r["tokens_per_second"] for r in model_results) / len(
312 | model_results
313 | )
314 | avg_time = sum(r["execution_time"] for r in model_results) / len(
315 | model_results
316 | )
317 | success_rate = (
318 | len([r for r in model_results if r["target_achieved"]])
319 | / len(model_results)
320 | ) * 100
321 |
322 | print(f" {model}:")
323 | print(f" Tests: {len(model_results)}")
324 | print(f" Avg Speed: {avg_speed:.0f} tok/s")
325 | print(f" Avg Time: {avg_time:.2f}s")
326 | print(f" Success Rate: {success_rate:.0f}%")
327 |
328 | async def run_comprehensive_demo(self):
329 | """Run the complete LLM speed demonstration."""
330 | print("🚀 MaverickMCP LLM Speed Optimization Demonstration")
331 | print(f"⏰ Started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
332 | print("🎯 Goal: Demonstrate 2-3x LLM speed improvements")
333 |
334 | # Step 1: Validate connection
335 | if not await self.validate_openrouter_connection():
336 | print("\n❌ Cannot proceed - API connection failed")
337 | return False
338 |
339 | # Step 2: Show model selection intelligence
340 | await self.demonstrate_model_selection()
341 |
342 | # Step 3: Run live speed benchmarks
343 | results = await self.run_speed_benchmarks()
344 |
345 | # Step 4: Analyze results
346 | self.analyze_benchmark_results(results)
347 |
348 | # Final summary
349 | self.print_header("🎉 DEMONSTRATION SUMMARY")
350 |
351 | successful_tests = [r for r in results if not r.get("error")]
352 | targets_achieved = [r for r in successful_tests if r.get("target_achieved")]
353 |
354 | print("✅ LLM Speed Optimization Results:")
355 | print(f" Tests Executed: {len(results)}")
356 | print(f" Successful: {len(successful_tests)}")
357 | print(f" Targets Achieved: {len(targets_achieved)}")
358 | print(f" Success Rate: {(len(targets_achieved) / len(results) * 100):.1f}%")
359 |
360 | if successful_tests:
361 | max_time = max(r["execution_time"] for r in successful_tests)
362 | avg_speed = sum(r["tokens_per_second"] for r in successful_tests) / len(
363 | successful_tests
364 | )
365 | speed_improvement = 60.0 / max_time if max_time > 0 else 0
366 |
367 | print(
368 | f" Fastest Response: {min(r['execution_time'] for r in successful_tests):.2f}s"
369 | )
370 | print(f" Average Token Speed: {avg_speed:.0f} tok/s")
371 | print(f" Speed Improvement: {speed_improvement:.1f}x faster")
372 |
373 | print("\n📊 Key Optimizations Demonstrated:")
374 | print(" ✅ Adaptive Model Selection (context-aware)")
375 | print(" ✅ Time-Budget Optimization")
376 | print(" ✅ Fast Model Utilization (Gemini Flash, Claude Haiku)")
377 | print(" ✅ Progressive Timeout Management")
378 | print(" ✅ Token Generation Speed Optimization")
379 |
380 | # Success criteria: at least 75% success rate and 2x improvement
381 | success_criteria = len(targets_achieved) >= len(results) * 0.75 and (
382 | successful_tests
383 | and 60.0 / max(r["execution_time"] for r in successful_tests) >= 2.0
384 | )
385 |
386 | return success_criteria
387 |
388 |
389 | async def main():
390 | """Main demonstration entry point."""
391 | demo = LLMSpeedDemonstrator()
392 |
393 | try:
394 | success = await demo.run_comprehensive_demo()
395 |
396 | if success:
397 | print("\n🎉 LLM Speed Demonstration PASSED - Optimizations validated!")
398 | return 0
399 | else:
400 | print("\n⚠️ Demonstration had mixed results - review analysis above")
401 | return 1
402 |
403 | except KeyboardInterrupt:
404 | print("\n\n⏹️ Demonstration interrupted by user")
405 | return 130
406 | except Exception as e:
407 | print(f"\n💥 Demonstration failed with error: {e}")
408 | import traceback
409 |
410 | traceback.print_exc()
411 | return 1
412 |
413 |
414 | if __name__ == "__main__":
415 | # Check required environment variables
416 | if not os.getenv("OPENROUTER_API_KEY"):
417 | print("❌ Missing OPENROUTER_API_KEY environment variable")
418 | print("Please check your .env file")
419 | sys.exit(1)
420 |
421 | # Run the demonstration
422 | exit_code = asyncio.run(main())
423 | sys.exit(exit_code)
424 |
```
--------------------------------------------------------------------------------
/maverick_mcp/utils/resource_manager.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Resource management utilities for the backtesting system.
3 | Handles memory limits, resource cleanup, and system resource monitoring.
4 | """
5 |
6 | import asyncio
7 | import logging
8 | import os
9 | import resource
10 | import signal
11 | import threading
12 | import time
13 | from collections.abc import Callable
14 | from contextlib import contextmanager
15 | from dataclasses import dataclass
16 | from typing import Any, Optional
17 |
18 | import psutil
19 |
20 | from maverick_mcp.utils.memory_profiler import (
21 | check_memory_leak,
22 | force_garbage_collection,
23 | get_memory_stats,
24 | )
25 |
26 | logger = logging.getLogger(__name__)
27 |
28 | # Resource limits (in bytes)
29 | DEFAULT_MEMORY_LIMIT = 2 * 1024 * 1024 * 1024 # 2GB
30 | DEFAULT_SWAP_LIMIT = 4 * 1024 * 1024 * 1024 # 4GB
31 | CRITICAL_MEMORY_THRESHOLD = 0.9 # 90% of limit
32 |
33 | # Global resource manager instance
34 | _resource_manager: Optional["ResourceManager"] = None
35 |
36 |
37 | @dataclass
38 | class ResourceLimits:
39 | """Resource limit configuration."""
40 |
41 | memory_limit_bytes: int = DEFAULT_MEMORY_LIMIT
42 | swap_limit_bytes: int = DEFAULT_SWAP_LIMIT
43 | cpu_time_limit_seconds: int = 3600 # 1 hour
44 | file_descriptor_limit: int = 1024
45 | enable_memory_monitoring: bool = True
46 | enable_cpu_monitoring: bool = True
47 | cleanup_interval_seconds: int = 60
48 |
49 |
50 | @dataclass
51 | class ResourceUsage:
52 | """Current resource usage snapshot."""
53 |
54 | memory_rss_bytes: int
55 | memory_vms_bytes: int
56 | memory_percent: float
57 | cpu_percent: float
58 | open_files: int
59 | threads: int
60 | timestamp: float
61 |
62 |
63 | class ResourceExhaustionError(Exception):
64 | """Raised when resource limits are exceeded."""
65 |
66 | pass
67 |
68 |
69 | class ResourceManager:
70 | """System resource manager with limits and cleanup."""
71 |
72 | def __init__(self, limits: ResourceLimits = None):
73 | """Initialize resource manager.
74 |
75 | Args:
76 | limits: Resource limits configuration
77 | """
78 | self.limits = limits or ResourceLimits()
79 | self.process = psutil.Process()
80 | self.monitoring_active = False
81 | self.cleanup_callbacks: list[Callable[[], None]] = []
82 | self.resource_history: list[ResourceUsage] = []
83 | self.max_history_size = 100
84 |
85 | # Setup signal handlers for graceful shutdown
86 | self._setup_signal_handlers()
87 |
88 | # Start monitoring if enabled
89 | if self.limits.enable_memory_monitoring or self.limits.enable_cpu_monitoring:
90 | self.start_monitoring()
91 |
92 | def _setup_signal_handlers(self):
93 | """Setup signal handlers for resource cleanup."""
94 |
95 | def signal_handler(signum, frame):
96 | logger.info(f"Received signal {signum}, performing cleanup")
97 | self.cleanup_all()
98 |
99 | signal.signal(signal.SIGTERM, signal_handler)
100 | signal.signal(signal.SIGINT, signal_handler)
101 |
102 | def start_monitoring(self):
103 | """Start resource monitoring in background thread."""
104 | if self.monitoring_active:
105 | return
106 |
107 | self.monitoring_active = True
108 | monitor_thread = threading.Thread(target=self._monitor_resources, daemon=True)
109 | monitor_thread.start()
110 | logger.info("Resource monitoring started")
111 |
112 | def stop_monitoring(self):
113 | """Stop resource monitoring."""
114 | self.monitoring_active = False
115 | logger.info("Resource monitoring stopped")
116 |
117 | def _monitor_resources(self):
118 | """Background resource monitoring loop."""
119 | while self.monitoring_active:
120 | try:
121 | usage = self.get_current_usage()
122 | self.resource_history.append(usage)
123 |
124 | # Keep history size manageable
125 | if len(self.resource_history) > self.max_history_size:
126 | self.resource_history.pop(0)
127 |
128 | # Check limits and trigger cleanup if needed
129 | self._check_resource_limits(usage)
130 |
131 | time.sleep(self.limits.cleanup_interval_seconds)
132 |
133 | except Exception as e:
134 | logger.error(f"Error in resource monitoring: {e}")
135 | time.sleep(30) # Back off on errors
136 |
137 | def get_current_usage(self) -> ResourceUsage:
138 | """Get current resource usage."""
139 | try:
140 | memory_info = self.process.memory_info()
141 | cpu_percent = self.process.cpu_percent()
142 |
143 | # Get open files count safely
144 | try:
145 | open_files = len(self.process.open_files())
146 | except (psutil.AccessDenied, psutil.NoSuchProcess):
147 | open_files = 0
148 |
149 | # Get thread count safely
150 | try:
151 | threads = self.process.num_threads()
152 | except (psutil.AccessDenied, psutil.NoSuchProcess):
153 | threads = 0
154 |
155 | return ResourceUsage(
156 | memory_rss_bytes=memory_info.rss,
157 | memory_vms_bytes=memory_info.vms,
158 | memory_percent=self.process.memory_percent(),
159 | cpu_percent=cpu_percent,
160 | open_files=open_files,
161 | threads=threads,
162 | timestamp=time.time(),
163 | )
164 |
165 | except Exception as e:
166 | logger.error(f"Error getting resource usage: {e}")
167 | return ResourceUsage(0, 0, 0, 0, 0, 0, time.time())
168 |
169 | def _check_resource_limits(self, usage: ResourceUsage):
170 | """Check if resource limits are exceeded and take action."""
171 | # Memory limit check
172 | if (
173 | usage.memory_rss_bytes
174 | > self.limits.memory_limit_bytes * CRITICAL_MEMORY_THRESHOLD
175 | ):
176 | logger.warning(
177 | f"Memory usage {usage.memory_rss_bytes / (1024**3):.2f}GB "
178 | f"approaching limit {self.limits.memory_limit_bytes / (1024**3):.2f}GB"
179 | )
180 | self._trigger_emergency_cleanup()
181 |
182 | if usage.memory_rss_bytes > self.limits.memory_limit_bytes:
183 | logger.critical(
184 | f"Memory limit exceeded: {usage.memory_rss_bytes / (1024**3):.2f}GB "
185 | f"> {self.limits.memory_limit_bytes / (1024**3):.2f}GB"
186 | )
187 | raise ResourceExhaustionError("Memory limit exceeded")
188 |
189 | # File descriptor check
190 | if usage.open_files > self.limits.file_descriptor_limit * 0.9:
191 | logger.warning(f"High file descriptor usage: {usage.open_files}")
192 | self._close_unused_files()
193 |
194 | def _trigger_emergency_cleanup(self):
195 | """Trigger emergency resource cleanup."""
196 | logger.info("Triggering emergency resource cleanup")
197 |
198 | # Force garbage collection
199 | force_garbage_collection()
200 |
201 | # Run cleanup callbacks
202 | for callback in self.cleanup_callbacks:
203 | try:
204 | callback()
205 | except Exception as e:
206 | logger.error(f"Error in cleanup callback: {e}")
207 |
208 | # Clear memory profiler snapshots
209 | try:
210 | from maverick_mcp.utils.memory_profiler import reset_memory_stats
211 |
212 | reset_memory_stats()
213 | except ImportError:
214 | pass
215 |
216 | # Clear cache if available
217 | try:
218 | from maverick_mcp.data.cache import clear_cache
219 |
220 | clear_cache()
221 | except ImportError:
222 | pass
223 |
224 | def _close_unused_files(self):
225 | """Close unused file descriptors."""
226 | try:
227 | # Get current open files
228 | open_files = self.process.open_files()
229 | logger.debug(f"Found {len(open_files)} open files")
230 |
231 | # Note: We can't automatically close files as that might break the application
232 | # This is mainly for monitoring and alerting
233 | for file_info in open_files:
234 | logger.debug(f"Open file: {file_info.path}")
235 |
236 | except Exception as e:
237 | logger.debug(f"Could not enumerate open files: {e}")
238 |
239 | def add_cleanup_callback(self, callback: Callable[[], None]):
240 | """Add a cleanup callback function."""
241 | self.cleanup_callbacks.append(callback)
242 |
243 | def cleanup_all(self):
244 | """Run all cleanup callbacks and garbage collection."""
245 | logger.info("Running comprehensive resource cleanup")
246 |
247 | # Run cleanup callbacks
248 | for callback in self.cleanup_callbacks:
249 | try:
250 | callback()
251 | except Exception as e:
252 | logger.error(f"Error in cleanup callback: {e}")
253 |
254 | # Force garbage collection
255 | force_garbage_collection()
256 |
257 | # Log final resource usage
258 | usage = self.get_current_usage()
259 | logger.info(
260 | f"Post-cleanup usage: {usage.memory_rss_bytes / (1024**2):.2f}MB memory, "
261 | f"{usage.open_files} files, {usage.threads} threads"
262 | )
263 |
264 | def get_resource_report(self) -> dict[str, Any]:
265 | """Get comprehensive resource usage report."""
266 | current = self.get_current_usage()
267 |
268 | report = {
269 | "current_usage": {
270 | "memory_mb": current.memory_rss_bytes / (1024**2),
271 | "memory_percent": current.memory_percent,
272 | "cpu_percent": current.cpu_percent,
273 | "open_files": current.open_files,
274 | "threads": current.threads,
275 | },
276 | "limits": {
277 | "memory_limit_mb": self.limits.memory_limit_bytes / (1024**2),
278 | "memory_usage_ratio": current.memory_rss_bytes
279 | / self.limits.memory_limit_bytes,
280 | "file_descriptor_limit": self.limits.file_descriptor_limit,
281 | },
282 | "monitoring": {
283 | "active": self.monitoring_active,
284 | "history_size": len(self.resource_history),
285 | "cleanup_callbacks": len(self.cleanup_callbacks),
286 | },
287 | }
288 |
289 | # Add memory profiler stats if available
290 | try:
291 | memory_stats = get_memory_stats()
292 | report["memory_profiler"] = memory_stats
293 | except Exception:
294 | pass
295 |
296 | return report
297 |
298 | def set_memory_limit(self, limit_bytes: int):
299 | """Set memory limit for the process."""
300 | try:
301 | # Set soft and hard memory limits
302 | resource.setrlimit(resource.RLIMIT_AS, (limit_bytes, limit_bytes))
303 | self.limits.memory_limit_bytes = limit_bytes
304 | logger.info(f"Memory limit set to {limit_bytes / (1024**3):.2f}GB")
305 | except Exception as e:
306 | logger.warning(f"Could not set memory limit: {e}")
307 |
308 | def check_memory_health(self) -> dict[str, Any]:
309 | """Check memory health and detect potential issues."""
310 | health_report = {
311 | "status": "healthy",
312 | "issues": [],
313 | "recommendations": [],
314 | }
315 |
316 | current = self.get_current_usage()
317 |
318 | # Check memory usage
319 | usage_ratio = current.memory_rss_bytes / self.limits.memory_limit_bytes
320 | if usage_ratio > 0.9:
321 | health_report["status"] = "critical"
322 | health_report["issues"].append(f"Memory usage at {usage_ratio:.1%}")
323 | health_report["recommendations"].append("Trigger immediate cleanup")
324 | elif usage_ratio > 0.7:
325 | health_report["status"] = "warning"
326 | health_report["issues"].append(f"High memory usage at {usage_ratio:.1%}")
327 | health_report["recommendations"].append("Consider cleanup")
328 |
329 | # Check for memory leaks
330 | if check_memory_leak(threshold_mb=100.0):
331 | health_report["status"] = "warning"
332 | health_report["issues"].append("Potential memory leak detected")
333 | health_report["recommendations"].append("Review memory profiler logs")
334 |
335 | # Check file descriptor usage
336 | fd_ratio = current.open_files / self.limits.file_descriptor_limit
337 | if fd_ratio > 0.8:
338 | health_report["status"] = "warning"
339 | health_report["issues"].append(
340 | f"High file descriptor usage: {current.open_files}"
341 | )
342 | health_report["recommendations"].append("Review open files")
343 |
344 | return health_report
345 |
346 |
347 | @contextmanager
348 | def resource_limit_context(
349 | memory_limit_mb: int = None,
350 | cpu_limit_percent: float = None,
351 | cleanup_on_exit: bool = True,
352 | ):
353 | """Context manager for resource-limited operations.
354 |
355 | Args:
356 | memory_limit_mb: Memory limit in MB
357 | cpu_limit_percent: CPU limit as percentage
358 | cleanup_on_exit: Whether to cleanup on exit
359 |
360 | Yields:
361 | ResourceManager instance
362 | """
363 | limits = ResourceLimits()
364 | if memory_limit_mb:
365 | limits.memory_limit_bytes = memory_limit_mb * 1024 * 1024
366 |
367 | manager = ResourceManager(limits)
368 |
369 | try:
370 | yield manager
371 | finally:
372 | if cleanup_on_exit:
373 | manager.cleanup_all()
374 | manager.stop_monitoring()
375 |
376 |
377 | def get_resource_manager() -> ResourceManager:
378 | """Get or create global resource manager instance."""
379 | global _resource_manager
380 | if _resource_manager is None:
381 | _resource_manager = ResourceManager()
382 | return _resource_manager
383 |
384 |
385 | def set_process_memory_limit(limit_gb: float):
386 | """Set memory limit for current process.
387 |
388 | Args:
389 | limit_gb: Memory limit in gigabytes
390 | """
391 | limit_bytes = int(limit_gb * 1024 * 1024 * 1024)
392 | manager = get_resource_manager()
393 | manager.set_memory_limit(limit_bytes)
394 |
395 |
396 | def monitor_async_task(task: asyncio.Task, name: str = "unknown"):
397 | """Monitor an async task for resource usage.
398 |
399 | Args:
400 | task: Asyncio task to monitor
401 | name: Name of the task for logging
402 | """
403 |
404 | def task_done_callback(finished_task):
405 | if finished_task.exception():
406 | logger.error(f"Task {name} failed: {finished_task.exception()}")
407 | else:
408 | logger.debug(f"Task {name} completed successfully")
409 |
410 | # Trigger cleanup
411 | manager = get_resource_manager()
412 | manager._trigger_emergency_cleanup()
413 |
414 | task.add_done_callback(task_done_callback)
415 |
416 |
417 | class ResourceAwareExecutor:
418 | """Executor that respects resource limits."""
419 |
420 | def __init__(self, max_workers: int = None, memory_limit_mb: int = None):
421 | """Initialize resource-aware executor.
422 |
423 | Args:
424 | max_workers: Maximum worker threads
425 | memory_limit_mb: Memory limit in MB
426 | """
427 | self.max_workers = max_workers or min(32, (os.cpu_count() or 1) + 4)
428 | self.memory_limit_mb = memory_limit_mb or 500
429 | self.active_tasks = 0
430 | self.lock = threading.Lock()
431 |
432 | def submit(self, fn: Callable, *args, **kwargs):
433 | """Submit a task for execution with resource monitoring."""
434 | with self.lock:
435 | if self.active_tasks >= self.max_workers:
436 | raise ResourceExhaustionError("Too many active tasks")
437 |
438 | # Check memory before starting
439 | current_usage = get_resource_manager().get_current_usage()
440 | if current_usage.memory_rss_bytes > self.memory_limit_mb * 1024 * 1024:
441 | raise ResourceExhaustionError("Memory limit would be exceeded")
442 |
443 | self.active_tasks += 1
444 |
445 | try:
446 | result = fn(*args, **kwargs)
447 | return result
448 | finally:
449 | with self.lock:
450 | self.active_tasks -= 1
451 |
452 |
453 | # Utility functions
454 |
455 |
456 | def cleanup_on_low_memory(threshold_mb: float = 500.0):
457 | """Decorator to trigger cleanup when memory is low.
458 |
459 | Args:
460 | threshold_mb: Memory threshold in MB
461 | """
462 |
463 | def decorator(func):
464 | def wrapper(*args, **kwargs):
465 | get_resource_manager().get_current_usage()
466 | available_mb = psutil.virtual_memory().available / (1024**2)
467 |
468 | if available_mb < threshold_mb:
469 | logger.warning(
470 | f"Low memory detected ({available_mb:.1f}MB), triggering cleanup"
471 | )
472 | get_resource_manager()._trigger_emergency_cleanup()
473 |
474 | return func(*args, **kwargs)
475 |
476 | return wrapper
477 |
478 | return decorator
479 |
480 |
481 | def log_resource_usage(func: Callable = None, *, interval: int = 60):
482 | """Decorator to log resource usage periodically.
483 |
484 | Args:
485 | func: Function to decorate
486 | interval: Logging interval in seconds
487 | """
488 |
489 | def decorator(f):
490 | def wrapper(*args, **kwargs):
491 | start_time = time.time()
492 | start_usage = get_resource_manager().get_current_usage()
493 |
494 | try:
495 | return f(*args, **kwargs)
496 | finally:
497 | end_usage = get_resource_manager().get_current_usage()
498 | duration = time.time() - start_time
499 |
500 | memory_delta = end_usage.memory_rss_bytes - start_usage.memory_rss_bytes
501 | logger.info(
502 | f"{f.__name__} completed in {duration:.2f}s, "
503 | f"memory delta: {memory_delta / (1024**2):+.2f}MB"
504 | )
505 |
506 | return wrapper
507 |
508 | if func is None:
509 | return decorator
510 | else:
511 | return decorator(func)
512 |
513 |
514 | # Initialize global resource manager
515 | def initialize_resource_management(memory_limit_gb: float = 2.0):
516 | """Initialize global resource management.
517 |
518 | Args:
519 | memory_limit_gb: Memory limit in GB
520 | """
521 | global _resource_manager
522 |
523 | limits = ResourceLimits(
524 | memory_limit_bytes=int(memory_limit_gb * 1024 * 1024 * 1024),
525 | enable_memory_monitoring=True,
526 | enable_cpu_monitoring=True,
527 | )
528 |
529 | _resource_manager = ResourceManager(limits)
530 | logger.info(
531 | f"Resource management initialized with {memory_limit_gb}GB memory limit"
532 | )
533 |
534 |
535 | # Cleanup function for graceful shutdown
536 | def shutdown_resource_management():
537 | """Shutdown resource management gracefully."""
538 | global _resource_manager
539 | if _resource_manager:
540 | _resource_manager.stop_monitoring()
541 | _resource_manager.cleanup_all()
542 | _resource_manager = None
543 | logger.info("Resource management shut down")
544 |
```