This is page 21 of 39. Use http://codebase.md/wshobson/maverick-mcp?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .dockerignore
├── .env.example
├── .github
│ ├── dependabot.yml
│ ├── FUNDING.yml
│ ├── ISSUE_TEMPLATE
│ │ ├── bug_report.md
│ │ ├── config.yml
│ │ ├── feature_request.md
│ │ ├── question.md
│ │ └── security_report.md
│ ├── pull_request_template.md
│ └── workflows
│ ├── claude-code-review.yml
│ └── claude.yml
├── .gitignore
├── .python-version
├── .vscode
│ ├── launch.json
│ └── settings.json
├── alembic
│ ├── env.py
│ ├── script.py.mako
│ └── versions
│ ├── 001_initial_schema.py
│ ├── 003_add_performance_indexes.py
│ ├── 006_rename_metadata_columns.py
│ ├── 008_performance_optimization_indexes.py
│ ├── 009_rename_to_supply_demand.py
│ ├── 010_self_contained_schema.py
│ ├── 011_remove_proprietary_terms.py
│ ├── 013_add_backtest_persistence_models.py
│ ├── 014_add_portfolio_models.py
│ ├── 08e3945a0c93_merge_heads.py
│ ├── 9374a5c9b679_merge_heads_for_testing.py
│ ├── abf9b9afb134_merge_multiple_heads.py
│ ├── adda6d3fd84b_merge_proprietary_terms_removal_with_.py
│ ├── e0c75b0bdadb_fix_financial_data_precision_only.py
│ ├── f0696e2cac15_add_essential_performance_indexes.py
│ └── fix_database_integrity_issues.py
├── alembic.ini
├── CLAUDE.md
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── DATABASE_SETUP.md
├── docker-compose.override.yml.example
├── docker-compose.yml
├── Dockerfile
├── docs
│ ├── api
│ │ └── backtesting.md
│ ├── BACKTESTING.md
│ ├── COST_BASIS_SPECIFICATION.md
│ ├── deep_research_agent.md
│ ├── exa_research_testing_strategy.md
│ ├── PORTFOLIO_PERSONALIZATION_PLAN.md
│ ├── PORTFOLIO.md
│ ├── SETUP_SELF_CONTAINED.md
│ └── speed_testing_framework.md
├── examples
│ ├── complete_speed_validation.py
│ ├── deep_research_integration.py
│ ├── llm_optimization_example.py
│ ├── llm_speed_demo.py
│ ├── monitoring_example.py
│ ├── parallel_research_example.py
│ ├── speed_optimization_demo.py
│ └── timeout_fix_demonstration.py
├── LICENSE
├── Makefile
├── MANIFEST.in
├── maverick_mcp
│ ├── __init__.py
│ ├── agents
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── circuit_breaker.py
│ │ ├── deep_research.py
│ │ ├── market_analysis.py
│ │ ├── optimized_research.py
│ │ ├── supervisor.py
│ │ └── technical_analysis.py
│ ├── api
│ │ ├── __init__.py
│ │ ├── api_server.py
│ │ ├── connection_manager.py
│ │ ├── dependencies
│ │ │ ├── __init__.py
│ │ │ ├── stock_analysis.py
│ │ │ └── technical_analysis.py
│ │ ├── error_handling.py
│ │ ├── inspector_compatible_sse.py
│ │ ├── inspector_sse.py
│ │ ├── middleware
│ │ │ ├── error_handling.py
│ │ │ ├── mcp_logging.py
│ │ │ ├── rate_limiting_enhanced.py
│ │ │ └── security.py
│ │ ├── openapi_config.py
│ │ ├── routers
│ │ │ ├── __init__.py
│ │ │ ├── agents.py
│ │ │ ├── backtesting.py
│ │ │ ├── data_enhanced.py
│ │ │ ├── data.py
│ │ │ ├── health_enhanced.py
│ │ │ ├── health_tools.py
│ │ │ ├── health.py
│ │ │ ├── intelligent_backtesting.py
│ │ │ ├── introspection.py
│ │ │ ├── mcp_prompts.py
│ │ │ ├── monitoring.py
│ │ │ ├── news_sentiment_enhanced.py
│ │ │ ├── performance.py
│ │ │ ├── portfolio.py
│ │ │ ├── research.py
│ │ │ ├── screening_ddd.py
│ │ │ ├── screening_parallel.py
│ │ │ ├── screening.py
│ │ │ ├── technical_ddd.py
│ │ │ ├── technical_enhanced.py
│ │ │ ├── technical.py
│ │ │ └── tool_registry.py
│ │ ├── server.py
│ │ ├── services
│ │ │ ├── __init__.py
│ │ │ ├── base_service.py
│ │ │ ├── market_service.py
│ │ │ ├── portfolio_service.py
│ │ │ ├── prompt_service.py
│ │ │ └── resource_service.py
│ │ ├── simple_sse.py
│ │ └── utils
│ │ ├── __init__.py
│ │ ├── insomnia_export.py
│ │ └── postman_export.py
│ ├── application
│ │ ├── __init__.py
│ │ ├── commands
│ │ │ └── __init__.py
│ │ ├── dto
│ │ │ ├── __init__.py
│ │ │ └── technical_analysis_dto.py
│ │ ├── queries
│ │ │ ├── __init__.py
│ │ │ └── get_technical_analysis.py
│ │ └── screening
│ │ ├── __init__.py
│ │ ├── dtos.py
│ │ └── queries.py
│ ├── backtesting
│ │ ├── __init__.py
│ │ ├── ab_testing.py
│ │ ├── analysis.py
│ │ ├── batch_processing_stub.py
│ │ ├── batch_processing.py
│ │ ├── model_manager.py
│ │ ├── optimization.py
│ │ ├── persistence.py
│ │ ├── retraining_pipeline.py
│ │ ├── strategies
│ │ │ ├── __init__.py
│ │ │ ├── base.py
│ │ │ ├── ml
│ │ │ │ ├── __init__.py
│ │ │ │ ├── adaptive.py
│ │ │ │ ├── ensemble.py
│ │ │ │ ├── feature_engineering.py
│ │ │ │ └── regime_aware.py
│ │ │ ├── ml_strategies.py
│ │ │ ├── parser.py
│ │ │ └── templates.py
│ │ ├── strategy_executor.py
│ │ ├── vectorbt_engine.py
│ │ └── visualization.py
│ ├── config
│ │ ├── __init__.py
│ │ ├── constants.py
│ │ ├── database_self_contained.py
│ │ ├── database.py
│ │ ├── llm_optimization_config.py
│ │ ├── logging_settings.py
│ │ ├── plotly_config.py
│ │ ├── security_utils.py
│ │ ├── security.py
│ │ ├── settings.py
│ │ ├── technical_constants.py
│ │ ├── tool_estimation.py
│ │ └── validation.py
│ ├── core
│ │ ├── __init__.py
│ │ ├── technical_analysis.py
│ │ └── visualization.py
│ ├── data
│ │ ├── __init__.py
│ │ ├── cache_manager.py
│ │ ├── cache.py
│ │ ├── django_adapter.py
│ │ ├── health.py
│ │ ├── models.py
│ │ ├── performance.py
│ │ ├── session_management.py
│ │ └── validation.py
│ ├── database
│ │ ├── __init__.py
│ │ ├── base.py
│ │ └── optimization.py
│ ├── dependencies.py
│ ├── domain
│ │ ├── __init__.py
│ │ ├── entities
│ │ │ ├── __init__.py
│ │ │ └── stock_analysis.py
│ │ ├── events
│ │ │ └── __init__.py
│ │ ├── portfolio.py
│ │ ├── screening
│ │ │ ├── __init__.py
│ │ │ ├── entities.py
│ │ │ ├── services.py
│ │ │ └── value_objects.py
│ │ ├── services
│ │ │ ├── __init__.py
│ │ │ └── technical_analysis_service.py
│ │ ├── stock_analysis
│ │ │ ├── __init__.py
│ │ │ └── stock_analysis_service.py
│ │ └── value_objects
│ │ ├── __init__.py
│ │ └── technical_indicators.py
│ ├── exceptions.py
│ ├── infrastructure
│ │ ├── __init__.py
│ │ ├── cache
│ │ │ └── __init__.py
│ │ ├── caching
│ │ │ ├── __init__.py
│ │ │ └── cache_management_service.py
│ │ ├── connection_manager.py
│ │ ├── data_fetching
│ │ │ ├── __init__.py
│ │ │ └── stock_data_service.py
│ │ ├── health
│ │ │ ├── __init__.py
│ │ │ └── health_checker.py
│ │ ├── persistence
│ │ │ ├── __init__.py
│ │ │ └── stock_repository.py
│ │ ├── providers
│ │ │ └── __init__.py
│ │ ├── screening
│ │ │ ├── __init__.py
│ │ │ └── repositories.py
│ │ └── sse_optimizer.py
│ ├── langchain_tools
│ │ ├── __init__.py
│ │ ├── adapters.py
│ │ └── registry.py
│ ├── logging_config.py
│ ├── memory
│ │ ├── __init__.py
│ │ └── stores.py
│ ├── monitoring
│ │ ├── __init__.py
│ │ ├── health_check.py
│ │ ├── health_monitor.py
│ │ ├── integration_example.py
│ │ ├── metrics.py
│ │ ├── middleware.py
│ │ └── status_dashboard.py
│ ├── providers
│ │ ├── __init__.py
│ │ ├── dependencies.py
│ │ ├── factories
│ │ │ ├── __init__.py
│ │ │ ├── config_factory.py
│ │ │ └── provider_factory.py
│ │ ├── implementations
│ │ │ ├── __init__.py
│ │ │ ├── cache_adapter.py
│ │ │ ├── macro_data_adapter.py
│ │ │ ├── market_data_adapter.py
│ │ │ ├── persistence_adapter.py
│ │ │ └── stock_data_adapter.py
│ │ ├── interfaces
│ │ │ ├── __init__.py
│ │ │ ├── cache.py
│ │ │ ├── config.py
│ │ │ ├── macro_data.py
│ │ │ ├── market_data.py
│ │ │ ├── persistence.py
│ │ │ └── stock_data.py
│ │ ├── llm_factory.py
│ │ ├── macro_data.py
│ │ ├── market_data.py
│ │ ├── mocks
│ │ │ ├── __init__.py
│ │ │ ├── mock_cache.py
│ │ │ ├── mock_config.py
│ │ │ ├── mock_macro_data.py
│ │ │ ├── mock_market_data.py
│ │ │ ├── mock_persistence.py
│ │ │ └── mock_stock_data.py
│ │ ├── openrouter_provider.py
│ │ ├── optimized_screening.py
│ │ ├── optimized_stock_data.py
│ │ └── stock_data.py
│ ├── README.md
│ ├── tests
│ │ ├── __init__.py
│ │ ├── README_INMEMORY_TESTS.md
│ │ ├── test_cache_debug.py
│ │ ├── test_fixes_validation.py
│ │ ├── test_in_memory_routers.py
│ │ ├── test_in_memory_server.py
│ │ ├── test_macro_data_provider.py
│ │ ├── test_mailgun_email.py
│ │ ├── test_market_calendar_caching.py
│ │ ├── test_mcp_tool_fixes_pytest.py
│ │ ├── test_mcp_tool_fixes.py
│ │ ├── test_mcp_tools.py
│ │ ├── test_models_functional.py
│ │ ├── test_server.py
│ │ ├── test_stock_data_enhanced.py
│ │ ├── test_stock_data_provider.py
│ │ └── test_technical_analysis.py
│ ├── tools
│ │ ├── __init__.py
│ │ ├── performance_monitoring.py
│ │ ├── portfolio_manager.py
│ │ ├── risk_management.py
│ │ └── sentiment_analysis.py
│ ├── utils
│ │ ├── __init__.py
│ │ ├── agent_errors.py
│ │ ├── batch_processing.py
│ │ ├── cache_warmer.py
│ │ ├── circuit_breaker_decorators.py
│ │ ├── circuit_breaker_services.py
│ │ ├── circuit_breaker.py
│ │ ├── data_chunking.py
│ │ ├── database_monitoring.py
│ │ ├── debug_utils.py
│ │ ├── fallback_strategies.py
│ │ ├── llm_optimization.py
│ │ ├── logging_example.py
│ │ ├── logging_init.py
│ │ ├── logging.py
│ │ ├── mcp_logging.py
│ │ ├── memory_profiler.py
│ │ ├── monitoring_middleware.py
│ │ ├── monitoring.py
│ │ ├── orchestration_logging.py
│ │ ├── parallel_research.py
│ │ ├── parallel_screening.py
│ │ ├── quick_cache.py
│ │ ├── resource_manager.py
│ │ ├── shutdown.py
│ │ ├── stock_helpers.py
│ │ ├── structured_logger.py
│ │ ├── tool_monitoring.py
│ │ ├── tracing.py
│ │ └── yfinance_pool.py
│ ├── validation
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── data.py
│ │ ├── middleware.py
│ │ ├── portfolio.py
│ │ ├── responses.py
│ │ ├── screening.py
│ │ └── technical.py
│ └── workflows
│ ├── __init__.py
│ ├── agents
│ │ ├── __init__.py
│ │ ├── market_analyzer.py
│ │ ├── optimizer_agent.py
│ │ ├── strategy_selector.py
│ │ └── validator_agent.py
│ ├── backtesting_workflow.py
│ └── state.py
├── PLANS.md
├── pyproject.toml
├── pyrightconfig.json
├── README.md
├── scripts
│ ├── dev.sh
│ ├── INSTALLATION_GUIDE.md
│ ├── load_example.py
│ ├── load_market_data.py
│ ├── load_tiingo_data.py
│ ├── migrate_db.py
│ ├── README_TIINGO_LOADER.md
│ ├── requirements_tiingo.txt
│ ├── run_stock_screening.py
│ ├── run-migrations.sh
│ ├── seed_db.py
│ ├── seed_sp500.py
│ ├── setup_database.sh
│ ├── setup_self_contained.py
│ ├── setup_sp500_database.sh
│ ├── test_seeded_data.py
│ ├── test_tiingo_loader.py
│ ├── tiingo_config.py
│ └── validate_setup.py
├── SECURITY.md
├── server.json
├── setup.py
├── tests
│ ├── __init__.py
│ ├── conftest.py
│ ├── core
│ │ └── test_technical_analysis.py
│ ├── data
│ │ └── test_portfolio_models.py
│ ├── domain
│ │ ├── conftest.py
│ │ ├── test_portfolio_entities.py
│ │ └── test_technical_analysis_service.py
│ ├── fixtures
│ │ └── orchestration_fixtures.py
│ ├── integration
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── README.md
│ │ ├── run_integration_tests.sh
│ │ ├── test_api_technical.py
│ │ ├── test_chaos_engineering.py
│ │ ├── test_config_management.py
│ │ ├── test_full_backtest_workflow_advanced.py
│ │ ├── test_full_backtest_workflow.py
│ │ ├── test_high_volume.py
│ │ ├── test_mcp_tools.py
│ │ ├── test_orchestration_complete.py
│ │ ├── test_portfolio_persistence.py
│ │ ├── test_redis_cache.py
│ │ ├── test_security_integration.py.disabled
│ │ └── vcr_setup.py
│ ├── performance
│ │ ├── __init__.py
│ │ ├── test_benchmarks.py
│ │ ├── test_load.py
│ │ ├── test_profiling.py
│ │ └── test_stress.py
│ ├── providers
│ │ └── test_stock_data_simple.py
│ ├── README.md
│ ├── test_agents_router_mcp.py
│ ├── test_backtest_persistence.py
│ ├── test_cache_management_service.py
│ ├── test_cache_serialization.py
│ ├── test_circuit_breaker.py
│ ├── test_database_pool_config_simple.py
│ ├── test_database_pool_config.py
│ ├── test_deep_research_functional.py
│ ├── test_deep_research_integration.py
│ ├── test_deep_research_parallel_execution.py
│ ├── test_error_handling.py
│ ├── test_event_loop_integrity.py
│ ├── test_exa_research_integration.py
│ ├── test_exception_hierarchy.py
│ ├── test_financial_search.py
│ ├── test_graceful_shutdown.py
│ ├── test_integration_simple.py
│ ├── test_langgraph_workflow.py
│ ├── test_market_data_async.py
│ ├── test_market_data_simple.py
│ ├── test_mcp_orchestration_functional.py
│ ├── test_ml_strategies.py
│ ├── test_optimized_research_agent.py
│ ├── test_orchestration_integration.py
│ ├── test_orchestration_logging.py
│ ├── test_orchestration_tools_simple.py
│ ├── test_parallel_research_integration.py
│ ├── test_parallel_research_orchestrator.py
│ ├── test_parallel_research_performance.py
│ ├── test_performance_optimizations.py
│ ├── test_production_validation.py
│ ├── test_provider_architecture.py
│ ├── test_rate_limiting_enhanced.py
│ ├── test_runner_validation.py
│ ├── test_security_comprehensive.py.disabled
│ ├── test_security_cors.py
│ ├── test_security_enhancements.py.disabled
│ ├── test_security_headers.py
│ ├── test_security_penetration.py
│ ├── test_session_management.py
│ ├── test_speed_optimization_validation.py
│ ├── test_stock_analysis_dependencies.py
│ ├── test_stock_analysis_service.py
│ ├── test_stock_data_fetching_service.py
│ ├── test_supervisor_agent.py
│ ├── test_supervisor_functional.py
│ ├── test_tool_estimation_config.py
│ ├── test_visualization.py
│ └── utils
│ ├── test_agent_errors.py
│ ├── test_logging.py
│ ├── test_parallel_screening.py
│ └── test_quick_cache.py
├── tools
│ ├── check_orchestration_config.py
│ ├── experiments
│ │ ├── validation_examples.py
│ │ └── validation_fixed.py
│ ├── fast_dev.sh
│ ├── hot_reload.py
│ ├── quick_test.py
│ └── templates
│ ├── new_router_template.py
│ ├── new_tool_template.py
│ ├── screening_strategy_template.py
│ └── test_template.py
└── uv.lock
```
# Files
--------------------------------------------------------------------------------
/examples/complete_speed_validation.py:
--------------------------------------------------------------------------------
```python
1 | #!/usr/bin/env python3
2 | """
3 | Complete Speed Optimization Validation for MaverickMCP
4 |
5 | This comprehensive demonstration validates all speed optimization improvements
6 | including LLM optimizations and simulated research workflows to prove
7 | 2-3x speed improvements over the previous 138s/129s timeout failures.
8 |
9 | Validates:
10 | - Adaptive model selection (Gemini Flash for speed)
11 | - Progressive timeout management
12 | - Token generation speed (100+ tok/s for emergency scenarios)
13 | - Research workflow optimizations
14 | - Early termination strategies
15 | - Overall system performance under time pressure
16 | """
17 |
18 | import asyncio
19 | import os
20 | import sys
21 | import time
22 | from datetime import datetime
23 | from typing import Any
24 |
25 | # Add the project root to Python path
26 | sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
27 |
28 | from maverick_mcp.providers.openrouter_provider import OpenRouterProvider, TaskType
29 | from maverick_mcp.utils.llm_optimization import AdaptiveModelSelector
30 |
31 |
32 | class CompleteSpeedValidator:
33 | """Complete validation of all speed optimization features."""
34 |
35 | def __init__(self):
36 | """Initialize the validation suite."""
37 | api_key = os.getenv("OPENROUTER_API_KEY")
38 | if not api_key:
39 | raise ValueError(
40 | "OPENROUTER_API_KEY environment variable is required. "
41 | "Please set it with your OpenRouter API key."
42 | )
43 | self.openrouter_provider = OpenRouterProvider(api_key=api_key)
44 | self.model_selector = AdaptiveModelSelector(self.openrouter_provider)
45 |
46 | # Validation scenarios representing real-world usage
47 | self.validation_scenarios = [
48 | {
49 | "name": "⚡ Emergency Market Alert",
50 | "description": "Critical market alert requiring immediate analysis",
51 | "time_budget": 20.0,
52 | "target": "Sub-20s response with high-speed models",
53 | "phases": [
54 | {
55 | "name": "Quick Analysis",
56 | "prompt": "URGENT: NVIDIA down 8% after hours. Immediate impact assessment for AI sector in 2-3 bullet points.",
57 | "task_type": TaskType.QUICK_ANSWER,
58 | "weight": 1.0,
59 | }
60 | ],
61 | },
62 | {
63 | "name": "📊 Technical Analysis Request",
64 | "description": "Standard technical analysis request",
65 | "time_budget": 35.0,
66 | "target": "Sub-35s with comprehensive analysis",
67 | "phases": [
68 | {
69 | "name": "Technical Analysis",
70 | "prompt": "Provide technical analysis for Tesla (TSLA): current RSI, MACD signal, support/resistance levels, and price target.",
71 | "task_type": TaskType.TECHNICAL_ANALYSIS,
72 | "weight": 1.0,
73 | }
74 | ],
75 | },
76 | {
77 | "name": "🔍 Multi-Phase Research Simulation",
78 | "description": "Simulated research workflow with multiple phases",
79 | "time_budget": 60.0,
80 | "target": "Sub-60s with intelligent phase management",
81 | "phases": [
82 | {
83 | "name": "Market Context",
84 | "prompt": "Federal Reserve policy impact on tech stocks - key points only.",
85 | "task_type": TaskType.MARKET_ANALYSIS,
86 | "weight": 0.3,
87 | },
88 | {
89 | "name": "Sentiment Analysis",
90 | "prompt": "Current market sentiment for technology sector based on recent earnings.",
91 | "task_type": TaskType.SENTIMENT_ANALYSIS,
92 | "weight": 0.3,
93 | },
94 | {
95 | "name": "Synthesis",
96 | "prompt": "Synthesize: Tech sector outlook considering Fed policy and earnings sentiment.",
97 | "task_type": TaskType.RESULT_SYNTHESIS,
98 | "weight": 0.4,
99 | },
100 | ],
101 | },
102 | {
103 | "name": "🧠 Complex Research Challenge",
104 | "description": "Complex multi-factor analysis under time pressure",
105 | "time_budget": 90.0,
106 | "target": "Sub-90s with intelligent optimization",
107 | "phases": [
108 | {
109 | "name": "Sector Analysis",
110 | "prompt": "Renewable energy investment landscape 2025: policy drivers, technology trends, key opportunities.",
111 | "task_type": TaskType.MARKET_ANALYSIS,
112 | "weight": 0.4,
113 | },
114 | {
115 | "name": "Risk Assessment",
116 | "prompt": "Risk factors for renewable energy investments: regulatory, technological, and market risks.",
117 | "task_type": TaskType.RISK_ASSESSMENT,
118 | "weight": 0.3,
119 | },
120 | {
121 | "name": "Investment Synthesis",
122 | "prompt": "Top 3 renewable energy investment themes for 2025 with risk-adjusted outlook.",
123 | "task_type": TaskType.RESULT_SYNTHESIS,
124 | "weight": 0.3,
125 | },
126 | ],
127 | },
128 | ]
129 |
130 | def print_header(self, title: str):
131 | """Print formatted header."""
132 | print("\n" + "=" * 80)
133 | print(f" {title}")
134 | print("=" * 80)
135 |
136 | def print_phase_header(self, title: str):
137 | """Print phase header."""
138 | print(f"\n--- {title} ---")
139 |
140 | async def validate_system_readiness(self) -> bool:
141 | """Validate system is ready for speed testing."""
142 | self.print_header("🔧 SYSTEM READINESS VALIDATION")
143 |
144 | try:
145 | # Test OpenRouter connection with fast model
146 | test_llm = self.openrouter_provider.get_llm(TaskType.QUICK_ANSWER)
147 |
148 | start_time = time.time()
149 | from langchain_core.messages import HumanMessage
150 |
151 | test_response = await asyncio.wait_for(
152 | test_llm.ainvoke([HumanMessage(content="System ready?")]), timeout=10.0
153 | )
154 | response_time = time.time() - start_time
155 |
156 | print("✅ OpenRouter API: Connected and responding")
157 | print(f" Test Response Time: {response_time:.2f}s")
158 | print(f" Response Length: {len(test_response.content)} chars")
159 | print(
160 | f" Estimated Speed: ~{len(test_response.content) // 4 / response_time:.0f} tok/s"
161 | )
162 |
163 | # Test model selector
164 | print("\n🧠 Model Selection Intelligence: Active")
165 | from maverick_mcp.providers.openrouter_provider import MODEL_PROFILES
166 |
167 | print(f" Available models: {len(MODEL_PROFILES)} profiles")
168 | print(" Speed optimization: Enabled")
169 |
170 | return True
171 |
172 | except Exception as e:
173 | print(f"❌ System readiness check failed: {e}")
174 | return False
175 |
176 | async def run_validation_scenario(self, scenario: dict[str, Any]) -> dict[str, Any]:
177 | """Run a complete validation scenario."""
178 |
179 | print(f"\n🚀 Scenario: {scenario['name']}")
180 | print(f" Description: {scenario['description']}")
181 | print(f" Time Budget: {scenario['time_budget']}s")
182 | print(f" Target: {scenario['target']}")
183 |
184 | scenario_start = time.time()
185 | phase_results = []
186 | total_tokens = 0
187 | total_response_length = 0
188 |
189 | # Calculate time budget per phase based on weights
190 | remaining_budget = scenario["time_budget"]
191 |
192 | for i, phase in enumerate(scenario["phases"]):
193 | phase_budget = remaining_budget * phase["weight"]
194 |
195 | print(f"\n Phase {i + 1}: {phase['name']} (Budget: {phase_budget:.1f}s)")
196 |
197 | try:
198 | # Get optimal model for this phase
199 | complexity = self.model_selector.calculate_task_complexity(
200 | content=phase["prompt"],
201 | task_type=phase["task_type"],
202 | )
203 |
204 | model_config = self.model_selector.select_model_for_time_budget(
205 | task_type=phase["task_type"],
206 | time_remaining_seconds=phase_budget,
207 | complexity_score=complexity,
208 | content_size_tokens=len(phase["prompt"]) // 4,
209 | )
210 |
211 | print(f" Selected Model: {model_config.model_id}")
212 | print(f" Max Timeout: {model_config.timeout_seconds}s")
213 |
214 | # Execute phase
215 | llm = self.openrouter_provider.get_llm(
216 | model_override=model_config.model_id,
217 | temperature=model_config.temperature,
218 | max_tokens=model_config.max_tokens,
219 | )
220 |
221 | phase_start = time.time()
222 | from langchain_core.messages import HumanMessage
223 |
224 | response = await asyncio.wait_for(
225 | llm.ainvoke([HumanMessage(content=phase["prompt"])]),
226 | timeout=model_config.timeout_seconds,
227 | )
228 | phase_time = time.time() - phase_start
229 |
230 | # Calculate metrics
231 | response_length = len(response.content)
232 | estimated_tokens = response_length // 4
233 | tokens_per_second = (
234 | estimated_tokens / phase_time if phase_time > 0 else 0
235 | )
236 |
237 | phase_result = {
238 | "name": phase["name"],
239 | "execution_time": phase_time,
240 | "budget_used_pct": (phase_time / phase_budget) * 100,
241 | "model_used": model_config.model_id,
242 | "tokens_per_second": tokens_per_second,
243 | "response_length": response_length,
244 | "success": True,
245 | "response_preview": response.content[:100] + "..."
246 | if len(response.content) > 100
247 | else response.content,
248 | }
249 |
250 | phase_results.append(phase_result)
251 | total_tokens += estimated_tokens
252 | total_response_length += response_length
253 |
254 | print(
255 | f" ✅ Completed: {phase_time:.2f}s ({phase_result['budget_used_pct']:.1f}% of budget)"
256 | )
257 | print(f" Speed: {tokens_per_second:.0f} tok/s")
258 |
259 | # Update remaining budget
260 | remaining_budget -= phase_time
261 |
262 | # Early termination if running out of time
263 | if remaining_budget < 5 and i < len(scenario["phases"]) - 1:
264 | print(
265 | f" ⚠️ Early termination triggered - {remaining_budget:.1f}s remaining"
266 | )
267 | break
268 |
269 | except Exception as e:
270 | print(f" ❌ Phase failed: {str(e)}")
271 | phase_results.append(
272 | {
273 | "name": phase["name"],
274 | "execution_time": 0,
275 | "success": False,
276 | "error": str(e),
277 | }
278 | )
279 |
280 | # Calculate scenario metrics
281 | total_execution_time = time.time() - scenario_start
282 | successful_phases = [p for p in phase_results if p.get("success", False)]
283 |
284 | scenario_result = {
285 | "scenario_name": scenario["name"],
286 | "total_execution_time": total_execution_time,
287 | "time_budget": scenario["time_budget"],
288 | "budget_utilization": (total_execution_time / scenario["time_budget"])
289 | * 100,
290 | "target_achieved": total_execution_time <= scenario["time_budget"],
291 | "phases_completed": len(successful_phases),
292 | "phases_total": len(scenario["phases"]),
293 | "average_speed": sum(
294 | p.get("tokens_per_second", 0) for p in successful_phases
295 | )
296 | / len(successful_phases)
297 | if successful_phases
298 | else 0,
299 | "total_response_length": total_response_length,
300 | "phase_results": phase_results,
301 | "early_termination": len(successful_phases) < len(scenario["phases"]),
302 | }
303 |
304 | # Print scenario summary
305 | status_icon = "✅" if scenario_result["target_achieved"] else "⚠️"
306 | early_icon = "🔄" if scenario_result["early_termination"] else ""
307 |
308 | print(
309 | f"\n {status_icon} {early_icon} Scenario Complete: {total_execution_time:.2f}s"
310 | )
311 | print(f" Budget Used: {scenario_result['budget_utilization']:.1f}%")
312 | print(
313 | f" Phases: {scenario_result['phases_completed']}/{scenario_result['phases_total']}"
314 | )
315 | print(f" Avg Speed: {scenario_result['average_speed']:.0f} tok/s")
316 |
317 | return scenario_result
318 |
319 | def analyze_validation_results(self, results: list[dict[str, Any]]):
320 | """Analyze complete validation results."""
321 | self.print_header("📊 COMPLETE SPEED VALIDATION ANALYSIS")
322 |
323 | successful_scenarios = [r for r in results if r["phases_completed"] > 0]
324 | targets_achieved = [r for r in successful_scenarios if r["target_achieved"]]
325 |
326 | print("📈 Overall Validation Results:")
327 | print(f" Total Scenarios: {len(results)}")
328 | print(f" Successful: {len(successful_scenarios)}")
329 | print(f" Targets Achieved: {len(targets_achieved)}")
330 | print(f" Success Rate: {(len(targets_achieved) / len(results) * 100):.1f}%")
331 |
332 | if successful_scenarios:
333 | # Speed improvement analysis
334 | historical_baseline = 130.0 # Average of 138s and 129s timeout failures
335 | max_execution_time = max(
336 | r["total_execution_time"] for r in successful_scenarios
337 | )
338 | avg_execution_time = sum(
339 | r["total_execution_time"] for r in successful_scenarios
340 | ) / len(successful_scenarios)
341 | overall_improvement = (
342 | historical_baseline / max_execution_time
343 | if max_execution_time > 0
344 | else 0
345 | )
346 | avg_improvement = (
347 | historical_baseline / avg_execution_time
348 | if avg_execution_time > 0
349 | else 0
350 | )
351 |
352 | print("\n🎯 Speed Improvement Validation:")
353 | print(f" Historical Baseline: {historical_baseline}s (timeout failures)")
354 | print(f" Current Max Time: {max_execution_time:.2f}s")
355 | print(f" Current Avg Time: {avg_execution_time:.2f}s")
356 | print(f" Max Speed Improvement: {overall_improvement:.1f}x")
357 | print(f" Avg Speed Improvement: {avg_improvement:.1f}x")
358 |
359 | # Validation status
360 | if overall_improvement >= 3.0:
361 | print(
362 | f" 🎉 OUTSTANDING: {overall_improvement:.1f}x speed improvement!"
363 | )
364 | elif overall_improvement >= 2.0:
365 | print(
366 | f" ✅ SUCCESS: {overall_improvement:.1f}x speed improvement achieved!"
367 | )
368 | elif overall_improvement >= 1.5:
369 | print(f" 👍 GOOD: {overall_improvement:.1f}x improvement")
370 | else:
371 | print(f" ⚠️ MARGINAL: {overall_improvement:.1f}x improvement")
372 |
373 | # Performance breakdown by scenario type
374 | self.print_phase_header("⚡ PERFORMANCE BY SCENARIO TYPE")
375 |
376 | for result in successful_scenarios:
377 | print(f" {result['scenario_name']}")
378 | print(f" Execution Time: {result['total_execution_time']:.2f}s")
379 | print(f" Budget Used: {result['budget_utilization']:.1f}%")
380 | print(f" Average Speed: {result['average_speed']:.0f} tok/s")
381 | print(
382 | f" Phases Completed: {result['phases_completed']}/{result['phases_total']}"
383 | )
384 |
385 | # Show fastest phase
386 | successful_phases = [
387 | p for p in result["phase_results"] if p.get("success", False)
388 | ]
389 | if successful_phases:
390 | fastest_phase = min(
391 | successful_phases, key=lambda x: x["execution_time"]
392 | )
393 | print(
394 | f" Fastest Phase: {fastest_phase['name']} ({fastest_phase['execution_time']:.2f}s, {fastest_phase['tokens_per_second']:.0f} tok/s)"
395 | )
396 |
397 | print("")
398 |
399 | # Model performance analysis
400 | self.print_phase_header("🧠 MODEL PERFORMANCE ANALYSIS")
401 |
402 | model_stats = {}
403 | for result in successful_scenarios:
404 | for phase in result["phase_results"]:
405 | if phase.get("success", False):
406 | model = phase.get("model_used", "unknown")
407 | if model not in model_stats:
408 | model_stats[model] = {"times": [], "speeds": [], "count": 0}
409 | model_stats[model]["times"].append(phase["execution_time"])
410 | model_stats[model]["speeds"].append(phase["tokens_per_second"])
411 | model_stats[model]["count"] += 1
412 |
413 | for model, stats in model_stats.items():
414 | avg_time = sum(stats["times"]) / len(stats["times"])
415 | avg_speed = sum(stats["speeds"]) / len(stats["speeds"])
416 |
417 | print(f" {model}:")
418 | print(f" Uses: {stats['count']} phases")
419 | print(f" Avg Time: {avg_time:.2f}s")
420 | print(f" Avg Speed: {avg_speed:.0f} tok/s")
421 |
422 | # Speed category
423 | if avg_speed >= 100:
424 | speed_category = "🚀 Ultra-fast"
425 | elif avg_speed >= 60:
426 | speed_category = "⚡ Fast"
427 | elif avg_speed >= 30:
428 | speed_category = "🔄 Moderate"
429 | else:
430 | speed_category = "🐌 Slow"
431 |
432 | print(f" Category: {speed_category}")
433 | print("")
434 |
435 | async def run_complete_validation(self):
436 | """Run the complete speed validation suite."""
437 | print("🚀 MaverickMCP Complete Speed Optimization Validation")
438 | print(f"⏰ Started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
439 | print(
440 | "🎯 Goal: Validate 2-3x speed improvements over 138s/129s timeout failures"
441 | )
442 | print("📋 Scope: LLM optimizations + research workflow simulations")
443 |
444 | # Step 1: System readiness
445 | if not await self.validate_system_readiness():
446 | print("\n❌ System not ready for validation")
447 | return False
448 |
449 | # Step 2: Run validation scenarios
450 | self.print_header("🔍 RUNNING VALIDATION SCENARIOS")
451 |
452 | results = []
453 | total_start_time = time.time()
454 |
455 | for i, scenario in enumerate(self.validation_scenarios, 1):
456 | print(f"\n{'=' * 60}")
457 | print(f"SCENARIO {i}/{len(self.validation_scenarios)}")
458 | print(f"{'=' * 60}")
459 |
460 | try:
461 | result = await self.run_validation_scenario(scenario)
462 | results.append(result)
463 |
464 | # Brief pause between scenarios
465 | await asyncio.sleep(1)
466 |
467 | except Exception as e:
468 | print(f"💥 Scenario failed: {e}")
469 | results.append(
470 | {
471 | "scenario_name": scenario["name"],
472 | "total_execution_time": 0,
473 | "phases_completed": 0,
474 | "target_achieved": False,
475 | "error": str(e),
476 | }
477 | )
478 |
479 | total_validation_time = time.time() - total_start_time
480 |
481 | # Step 3: Analyze results
482 | self.analyze_validation_results(results)
483 |
484 | # Final validation summary
485 | self.print_header("🎉 VALIDATION COMPLETE")
486 |
487 | successful_scenarios = [r for r in results if r["phases_completed"] > 0]
488 | targets_achieved = [r for r in successful_scenarios if r["target_achieved"]]
489 |
490 | print("✅ Complete Speed Validation Results:")
491 | print(f" Scenarios Run: {len(results)}")
492 | print(f" Successful: {len(successful_scenarios)}")
493 | print(f" Targets Achieved: {len(targets_achieved)}")
494 | print(f" Success Rate: {(len(targets_achieved) / len(results) * 100):.1f}%")
495 | print(f" Total Validation Time: {total_validation_time:.2f}s")
496 |
497 | if successful_scenarios:
498 | max_time = max(r["total_execution_time"] for r in successful_scenarios)
499 | speed_improvement = 130.0 / max_time if max_time > 0 else 0
500 | print(f" Speed Improvement Achieved: {speed_improvement:.1f}x")
501 |
502 | print("\n📊 Optimizations Validated:")
503 | print(" ✅ Adaptive Model Selection (Gemini Flash for speed scenarios)")
504 | print(" ✅ Progressive Time Budget Management")
505 | print(" ✅ Early Termination Under Time Pressure")
506 | print(" ✅ Multi-Phase Workflow Optimization")
507 | print(" ✅ Token Generation Speed Optimization (100+ tok/s)")
508 | print(" ✅ Intelligent Timeout Management")
509 |
510 | # Success criteria: 75% success rate and 2x improvement
511 | validation_passed = (
512 | len(targets_achieved) >= len(results) * 0.75
513 | and successful_scenarios
514 | and 130.0 / max(r["total_execution_time"] for r in successful_scenarios)
515 | >= 1.8
516 | )
517 |
518 | return validation_passed
519 |
520 |
521 | async def main():
522 | """Main validation entry point."""
523 | validator = CompleteSpeedValidator()
524 |
525 | try:
526 | validation_passed = await validator.run_complete_validation()
527 |
528 | if validation_passed:
529 | print(
530 | "\n🎉 VALIDATION PASSED - Speed optimizations successfully validated!"
531 | )
532 | print(
533 | " System demonstrates 2-3x speed improvements over historical timeouts"
534 | )
535 | return 0
536 | else:
537 | print(
538 | "\n⚠️ VALIDATION MIXED RESULTS - Review analysis for improvement areas"
539 | )
540 | return 1
541 |
542 | except KeyboardInterrupt:
543 | print("\n\n⏹️ Validation interrupted by user")
544 | return 130
545 | except Exception as e:
546 | print(f"\n💥 Validation failed with error: {e}")
547 | import traceback
548 |
549 | traceback.print_exc()
550 | return 1
551 |
552 |
553 | if __name__ == "__main__":
554 | # Check required environment variables
555 | if not os.getenv("OPENROUTER_API_KEY"):
556 | print("❌ Missing OPENROUTER_API_KEY environment variable")
557 | print("Please check your .env file")
558 | sys.exit(1)
559 |
560 | # Run the complete validation
561 | exit_code = asyncio.run(main())
562 | sys.exit(exit_code)
563 |
```
--------------------------------------------------------------------------------
/maverick_mcp/agents/base.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Base classes for persona-aware agents using LangGraph best practices.
3 | """
4 |
5 | import logging
6 | from abc import ABC, abstractmethod
7 | from collections.abc import Sequence
8 | from datetime import datetime
9 | from typing import Annotated, Any
10 |
11 | from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
12 | from langchain_core.tools import BaseTool
13 | from langgraph.checkpoint.memory import MemorySaver
14 | from langgraph.graph import END, START, StateGraph, add_messages
15 | from langgraph.prebuilt import ToolNode
16 | from pydantic import BaseModel, Field
17 | from typing_extensions import TypedDict
18 |
19 | from maverick_mcp.config.settings import get_settings
20 |
21 | logger = logging.getLogger(__name__)
22 | settings = get_settings()
23 |
24 |
25 | class InvestorPersona(BaseModel):
26 | """Defines an investor persona with risk parameters."""
27 |
28 | name: str
29 | risk_tolerance: tuple[int, int] = Field(
30 | description="Risk tolerance range (min, max) on 0-100 scale"
31 | )
32 | position_size_max: float = Field(
33 | description="Maximum position size as percentage of portfolio"
34 | )
35 | stop_loss_multiplier: float = Field(
36 | description="Multiplier for stop loss calculation"
37 | )
38 | preferred_timeframe: str = Field(
39 | default="swing", description="Preferred trading timeframe: day, swing, position"
40 | )
41 | characteristics: list[str] = Field(
42 | default_factory=list, description="Key behavioral characteristics"
43 | )
44 |
45 |
46 | # Predefined investor personas
47 | INVESTOR_PERSONAS = {
48 | "conservative": InvestorPersona(
49 | name="Conservative",
50 | risk_tolerance=(
51 | settings.financial.risk_tolerance_conservative_min,
52 | settings.financial.risk_tolerance_conservative_max,
53 | ),
54 | position_size_max=settings.financial.max_position_size_conservative,
55 | stop_loss_multiplier=settings.financial.stop_loss_multiplier_conservative,
56 | preferred_timeframe="position",
57 | characteristics=[
58 | "Prioritizes capital preservation",
59 | "Focuses on dividend stocks",
60 | "Prefers established companies",
61 | "Long-term oriented",
62 | ],
63 | ),
64 | "moderate": InvestorPersona(
65 | name="Moderate",
66 | risk_tolerance=(
67 | settings.financial.risk_tolerance_moderate_min,
68 | settings.financial.risk_tolerance_moderate_max,
69 | ),
70 | position_size_max=settings.financial.max_position_size_moderate,
71 | stop_loss_multiplier=settings.financial.stop_loss_multiplier_moderate,
72 | preferred_timeframe="swing",
73 | characteristics=[
74 | "Balanced risk/reward approach",
75 | "Mix of growth and value",
76 | "Diversified portfolio",
77 | "Medium-term focus",
78 | ],
79 | ),
80 | "aggressive": InvestorPersona(
81 | name="Aggressive",
82 | risk_tolerance=(
83 | settings.financial.risk_tolerance_aggressive_min,
84 | settings.financial.risk_tolerance_aggressive_max,
85 | ),
86 | position_size_max=settings.financial.max_position_size_aggressive,
87 | stop_loss_multiplier=settings.financial.stop_loss_multiplier_aggressive,
88 | preferred_timeframe="day",
89 | characteristics=[
90 | "High risk tolerance",
91 | "Growth-focused",
92 | "Momentum trading",
93 | "Short-term opportunities",
94 | ],
95 | ),
96 | "day_trader": InvestorPersona(
97 | name="Day Trader",
98 | risk_tolerance=(
99 | settings.financial.risk_tolerance_day_trader_min,
100 | settings.financial.risk_tolerance_day_trader_max,
101 | ),
102 | position_size_max=settings.financial.max_position_size_day_trader,
103 | stop_loss_multiplier=settings.financial.stop_loss_multiplier_day_trader,
104 | preferred_timeframe="day",
105 | characteristics=[
106 | "Intraday positions only",
107 | "High-frequency trading",
108 | "Technical analysis focused",
109 | "Tight risk controls",
110 | ],
111 | ),
112 | }
113 |
114 |
115 | class BaseAgentState(TypedDict):
116 | """Base state for all persona-aware agents."""
117 |
118 | messages: Annotated[Sequence[BaseMessage], add_messages]
119 | persona: str
120 | session_id: str
121 |
122 |
123 | class PersonaAwareTool(BaseTool):
124 | """Base class for tools that adapt to investor personas."""
125 |
126 | persona: InvestorPersona | None = None
127 | # State tracking
128 | last_analysis_time: dict[str, datetime] = {}
129 | analyzed_stocks: dict[str, dict] = {}
130 | key_price_levels: dict[str, dict] = {}
131 | # Cache settings
132 | cache_ttl: int = settings.agent.agent_cache_ttl_seconds
133 |
134 | def set_persona(self, persona: InvestorPersona) -> None:
135 | """Set the active investor persona."""
136 | self.persona = persona
137 |
138 | def adjust_for_risk(self, value: float, parameter_type: str) -> float:
139 | """Adjust a value based on the persona's risk profile."""
140 | if not self.persona:
141 | return value
142 |
143 | # Get average risk tolerance
144 | risk_avg = sum(self.persona.risk_tolerance) / 2
145 | risk_factor = risk_avg / 50 # Normalize to 1.0 at moderate risk
146 |
147 | # Adjust based on parameter type
148 | if parameter_type == "position_size":
149 | # Kelly Criterion-inspired sizing
150 | kelly_fraction = self._calculate_kelly_fraction(risk_factor)
151 | adjusted = value * kelly_fraction
152 | return min(adjusted, self.persona.position_size_max)
153 | elif parameter_type == "stop_loss":
154 | # ATR-based dynamic stops
155 | return value * self.persona.stop_loss_multiplier
156 | elif parameter_type == "profit_target":
157 | # Risk-adjusted targets
158 | return value * (2 - risk_factor) # Conservative = lower targets
159 | elif parameter_type == "volatility_filter":
160 | # Volatility tolerance
161 | return value * (2 - risk_factor) # Conservative = lower vol tolerance
162 | elif parameter_type == "time_horizon":
163 | # Holding period in days
164 | if self.persona.preferred_timeframe == "day":
165 | return 1
166 | elif self.persona.preferred_timeframe == "swing":
167 | return 5 * risk_factor # 2.5-7.5 days
168 | else: # position
169 | return 20 * risk_factor # 10-30 days
170 | else:
171 | return value
172 |
173 | def _calculate_kelly_fraction(self, risk_factor: float) -> float:
174 | """Calculate position size using Kelly Criterion."""
175 | # Simplified Kelly: f = (p*b - q) / b
176 | # where p = win probability, b = win/loss ratio, q = loss probability
177 | # Using risk factor to adjust expected win rate
178 | win_probability = 0.45 + (0.1 * risk_factor) # 45-55% base win rate
179 | win_loss_ratio = 2.0 # 2:1 reward/risk
180 | loss_probability = 1 - win_probability
181 |
182 | kelly = (win_probability * win_loss_ratio - loss_probability) / win_loss_ratio
183 |
184 | # Apply safety factor (never use full Kelly)
185 | safety_factor = 0.25 # Use 25% of Kelly
186 | return max(0, kelly * safety_factor)
187 |
188 | def update_analysis_data(self, symbol: str, analysis_data: dict[str, Any]):
189 | """Update stored analysis data for a symbol."""
190 | symbol = symbol.upper()
191 | self.analyzed_stocks[symbol] = analysis_data
192 | self.last_analysis_time[symbol] = datetime.now()
193 | if "price_levels" in analysis_data:
194 | self.key_price_levels[symbol] = analysis_data["price_levels"]
195 |
196 | def get_stock_context(self, symbol: str) -> dict[str, Any]:
197 | """Get stored context for a symbol."""
198 | symbol = symbol.upper()
199 | return {
200 | "analysis": self.analyzed_stocks.get(symbol, {}),
201 | "last_analysis": self.last_analysis_time.get(symbol),
202 | "price_levels": self.key_price_levels.get(symbol, {}),
203 | "cache_expired": self._is_cache_expired(symbol),
204 | }
205 |
206 | def _is_cache_expired(self, symbol: str) -> bool:
207 | """Check if cached data has expired."""
208 | last_time = self.last_analysis_time.get(symbol.upper())
209 | if not last_time:
210 | return True
211 |
212 | age_seconds = (datetime.now() - last_time).total_seconds()
213 | return age_seconds > self.cache_ttl
214 |
215 | def _adjust_risk_parameters(self, params: dict) -> dict:
216 | """Adjust parameters based on risk profile."""
217 | if not self.persona:
218 | return params
219 |
220 | risk_factor = sum(self.persona.risk_tolerance) / 100 # 0.1-0.9 scale
221 |
222 | # Apply risk adjustments based on parameter names
223 | adjusted = {}
224 | for key, value in params.items():
225 | if isinstance(value, int | float):
226 | key_lower = key.lower()
227 | if any(term in key_lower for term in ["stop", "support", "risk"]):
228 | # Wider stops/support for conservative, tighter for aggressive
229 | adjusted[key] = value * (2 - risk_factor)
230 | elif any(
231 | term in key_lower for term in ["resistance", "target", "profit"]
232 | ):
233 | # Lower targets for conservative, higher for aggressive
234 | adjusted[key] = value * risk_factor
235 | elif any(term in key_lower for term in ["size", "amount", "shares"]):
236 | # Smaller positions for conservative, larger for aggressive
237 | adjusted[key] = self.adjust_for_risk(value, "position_size")
238 | elif any(term in key_lower for term in ["volume", "liquidity"]):
239 | # Higher liquidity requirements for conservative
240 | adjusted[key] = value * (2 - risk_factor)
241 | elif any(term in key_lower for term in ["volatility", "atr", "std"]):
242 | # Lower volatility tolerance for conservative
243 | adjusted[key] = self.adjust_for_risk(value, "volatility_filter")
244 | else:
245 | adjusted[key] = value
246 | else:
247 | adjusted[key] = value
248 |
249 | return adjusted
250 |
251 | def _validate_risk_levels(self, data: dict) -> bool:
252 | """Validate if the data meets the persona's risk criteria."""
253 | if not self.persona:
254 | return True
255 |
256 | min_risk, max_risk = self.persona.risk_tolerance
257 |
258 | # Extract risk metrics
259 | volatility = data.get("volatility", 0)
260 | beta = data.get("beta", 1.0)
261 |
262 | # Convert to risk score (0-100)
263 | volatility_score = min(100, volatility * 2) # Assume 50% vol = 100 risk
264 | beta_score = abs(beta - 1) * 100 # Distance from market
265 |
266 | # Combined risk score
267 | risk_score = (volatility_score + beta_score) / 2
268 |
269 | if risk_score < min_risk or risk_score > max_risk:
270 | return False
271 |
272 | # Persona-specific validations
273 | if self.persona.name == "Conservative":
274 | # Additional checks for conservative investors
275 | if data.get("debt_to_equity", 0) > 1.5:
276 | return False
277 | if data.get("current_ratio", 0) < 1.5:
278 | return False
279 | if data.get("dividend_yield", 0) < 0.02: # Prefer dividend stocks
280 | return False
281 | elif self.persona.name == "Day Trader":
282 | # Day traders need high liquidity
283 | if data.get("average_volume", 0) < 1_000_000:
284 | return False
285 | if data.get("spread_percentage", 0) > 0.1: # Tight spreads only
286 | return False
287 |
288 | return True
289 |
290 | def format_for_persona(self, data: dict) -> dict:
291 | """Format output data based on persona preferences."""
292 | if not self.persona:
293 | return data
294 |
295 | formatted = data.copy()
296 |
297 | # Add persona-specific insights
298 | formatted["persona_insights"] = {
299 | "suitable_for_profile": self._validate_risk_levels(data),
300 | "risk_adjusted_parameters": self._adjust_risk_parameters(
301 | data.get("parameters", {})
302 | ),
303 | "recommended_timeframe": self.persona.preferred_timeframe,
304 | "max_position_size": self.persona.position_size_max,
305 | }
306 |
307 | # Add risk warnings if needed
308 | warnings = []
309 | if not self._validate_risk_levels(data):
310 | warnings.append(f"Risk profile outside {self.persona.name} parameters")
311 |
312 | if data.get("volatility", 0) > 50:
313 | warnings.append("High volatility - consider smaller position size")
314 |
315 | if warnings:
316 | formatted["risk_warnings"] = warnings
317 |
318 | return formatted
319 |
320 |
321 | class PersonaAwareAgent(ABC):
322 | """
323 | Base class for agents that adapt behavior based on investor personas.
324 |
325 | This follows LangGraph best practices:
326 | - Uses StateGraph for workflow definition
327 | - Implements proper node/edge patterns
328 | - Supports native streaming modes
329 | - Uses TypedDict for state management
330 | """
331 |
332 | def __init__(
333 | self,
334 | llm,
335 | tools: list[BaseTool],
336 | persona: str = "moderate",
337 | checkpointer: MemorySaver | None = None,
338 | ttl_hours: int = 1,
339 | ):
340 | """
341 | Initialize a persona-aware agent.
342 |
343 | Args:
344 | llm: Language model to use
345 | tools: List of tools available to the agent
346 | persona: Investor persona name
347 | checkpointer: Optional checkpointer (defaults to MemorySaver)
348 | ttl_hours: Time-to-live for memory in hours
349 | """
350 | self.llm = llm
351 | self.tools = tools
352 | self.persona = INVESTOR_PERSONAS.get(persona, INVESTOR_PERSONAS["moderate"])
353 | self.ttl_hours = ttl_hours
354 |
355 | # Set up checkpointing
356 | if checkpointer is None:
357 | self.checkpointer = MemorySaver()
358 | else:
359 | self.checkpointer = checkpointer
360 |
361 | # Configure tools with persona
362 | for tool in self.tools:
363 | if isinstance(tool, PersonaAwareTool):
364 | tool.set_persona(self.persona)
365 |
366 | # Build the graph
367 | self.graph = self._build_graph()
368 |
369 | # Track usage
370 | self.total_tokens = 0
371 | self.conversation_start = datetime.now()
372 |
373 | def _build_graph(self):
374 | """Build the LangGraph workflow."""
375 | # Create the graph builder
376 | workflow = StateGraph(self.get_state_schema())
377 |
378 | # Add the agent node
379 | workflow.add_node("agent", self._agent_node)
380 |
381 | # Create tool node if tools are available
382 | if self.tools:
383 | tool_node = ToolNode(self.tools)
384 | workflow.add_node("tools", tool_node)
385 |
386 | # Add conditional edge from agent
387 | workflow.add_conditional_edges(
388 | "agent",
389 | self._should_continue,
390 | {
391 | # If agent returns tool calls, route to tools
392 | "continue": "tools",
393 | # Otherwise end
394 | "end": END,
395 | },
396 | )
397 |
398 | # Add edge from tools back to agent
399 | workflow.add_edge("tools", "agent")
400 | else:
401 | # No tools, just end after agent
402 | workflow.add_edge("agent", END)
403 |
404 | # Set entry point
405 | workflow.add_edge(START, "agent")
406 |
407 | # Compile with checkpointer
408 | return workflow.compile(checkpointer=self.checkpointer)
409 |
410 | def _agent_node(self, state: dict[str, Any]) -> dict[str, Any]:
411 | """The main agent node that processes messages."""
412 | messages = state["messages"]
413 |
414 | # Add system message if it's the first message
415 | if len(messages) == 1 and isinstance(messages[0], HumanMessage):
416 | system_prompt = self._build_system_prompt()
417 | messages = [SystemMessage(content=system_prompt)] + messages
418 |
419 | # Call the LLM
420 | if self.tools:
421 | response = self.llm.bind_tools(self.tools).invoke(messages)
422 | else:
423 | response = self.llm.invoke(messages)
424 |
425 | # Track tokens (simplified)
426 | if hasattr(response, "content"):
427 | self.total_tokens += len(response.content) // 4
428 |
429 | # Return the response
430 | return {"messages": [response]}
431 |
432 | def _should_continue(self, state: dict[str, Any]) -> str:
433 | """Determine whether to continue to tools or end."""
434 | last_message = state["messages"][-1]
435 |
436 | # If the LLM makes a tool call, continue to tools
437 | if hasattr(last_message, "tool_calls") and last_message.tool_calls:
438 | return "continue"
439 |
440 | # Otherwise we're done
441 | return "end"
442 |
443 | def _build_system_prompt(self) -> str:
444 | """Build system prompt based on persona."""
445 | base_prompt = f"""You are a financial advisor configured for a {self.persona.name} investor profile.
446 |
447 | Risk Parameters:
448 | - Risk Tolerance: {self.persona.risk_tolerance[0]}-{self.persona.risk_tolerance[1]}/100
449 | - Max Position Size: {self.persona.position_size_max * 100:.1f}% of portfolio
450 | - Stop Loss Multiplier: {self.persona.stop_loss_multiplier}x
451 | - Preferred Timeframe: {self.persona.preferred_timeframe}
452 |
453 | Key Characteristics:
454 | {chr(10).join(f"- {char}" for char in self.persona.characteristics)}
455 |
456 | Always adjust your recommendations to match this risk profile. Be explicit about risk management."""
457 |
458 | return base_prompt
459 |
460 | @abstractmethod
461 | def get_state_schema(self) -> type:
462 | """
463 | Get the state schema for this agent.
464 |
465 | Subclasses should return their specific state schema.
466 | """
467 | return BaseAgentState
468 |
469 | async def ainvoke(self, query: str, session_id: str, **kwargs) -> dict[str, Any]:
470 | """
471 | Invoke the agent asynchronously.
472 |
473 | Args:
474 | query: User query
475 | session_id: Session identifier
476 | **kwargs: Additional parameters
477 |
478 | Returns:
479 | Agent response
480 | """
481 | config = {
482 | "configurable": {"thread_id": session_id, "persona": self.persona.name}
483 | }
484 |
485 | # Merge additional config
486 | if "config" in kwargs:
487 | config.update(kwargs["config"])
488 |
489 | # Run the graph
490 | result = await self.graph.ainvoke(
491 | {
492 | "messages": [HumanMessage(content=query)],
493 | "persona": self.persona.name,
494 | "session_id": session_id,
495 | },
496 | config=config,
497 | )
498 |
499 | return self._extract_response(result)
500 |
501 | def invoke(self, query: str, session_id: str, **kwargs) -> dict[str, Any]:
502 | """
503 | Invoke the agent synchronously.
504 |
505 | Args:
506 | query: User query
507 | session_id: Session identifier
508 | **kwargs: Additional parameters
509 |
510 | Returns:
511 | Agent response
512 | """
513 | config = {
514 | "configurable": {"thread_id": session_id, "persona": self.persona.name}
515 | }
516 |
517 | # Merge additional config
518 | if "config" in kwargs:
519 | config.update(kwargs["config"])
520 |
521 | # Run the graph
522 | result = self.graph.invoke(
523 | {
524 | "messages": [HumanMessage(content=query)],
525 | "persona": self.persona.name,
526 | "session_id": session_id,
527 | },
528 | config=config,
529 | )
530 |
531 | return self._extract_response(result)
532 |
533 | async def astream(
534 | self, query: str, session_id: str, stream_mode: str = "values", **kwargs
535 | ):
536 | """
537 | Stream agent responses asynchronously.
538 |
539 | Args:
540 | query: User query
541 | session_id: Session identifier
542 | stream_mode: Streaming mode (values, updates, messages, custom, debug)
543 | **kwargs: Additional parameters
544 |
545 | Yields:
546 | Streamed chunks based on mode
547 | """
548 | config = {
549 | "configurable": {"thread_id": session_id, "persona": self.persona.name}
550 | }
551 |
552 | # Merge additional config
553 | if "config" in kwargs:
554 | config.update(kwargs["config"])
555 |
556 | # Stream the graph
557 | async for chunk in self.graph.astream(
558 | {
559 | "messages": [HumanMessage(content=query)],
560 | "persona": self.persona.name,
561 | "session_id": session_id,
562 | },
563 | config=config,
564 | stream_mode=stream_mode,
565 | ):
566 | yield chunk
567 |
568 | def stream(
569 | self, query: str, session_id: str, stream_mode: str = "values", **kwargs
570 | ):
571 | """
572 | Stream agent responses synchronously.
573 |
574 | Args:
575 | query: User query
576 | session_id: Session identifier
577 | stream_mode: Streaming mode (values, updates, messages, custom, debug)
578 | **kwargs: Additional parameters
579 |
580 | Yields:
581 | Streamed chunks based on mode
582 | """
583 | config = {
584 | "configurable": {"thread_id": session_id, "persona": self.persona.name}
585 | }
586 |
587 | # Merge additional config
588 | if "config" in kwargs:
589 | config.update(kwargs["config"])
590 |
591 | # Stream the graph
592 | yield from self.graph.stream(
593 | {
594 | "messages": [HumanMessage(content=query)],
595 | "persona": self.persona.name,
596 | "session_id": session_id,
597 | },
598 | config=config,
599 | stream_mode=stream_mode,
600 | )
601 |
602 | def _extract_response(self, result: dict[str, Any]) -> dict[str, Any]:
603 | """Extract the final response from graph execution."""
604 | messages = result.get("messages", [])
605 |
606 | if not messages:
607 | return {"content": "No response generated", "status": "error"}
608 |
609 | # Get the last AI message
610 | last_message = messages[-1]
611 |
612 | return {
613 | "content": last_message.content
614 | if hasattr(last_message, "content")
615 | else str(last_message),
616 | "status": "success",
617 | "persona": self.persona.name,
618 | "message_count": len(messages),
619 | "session_id": result.get("session_id", ""),
620 | }
621 |
622 | def get_risk_adjusted_params(
623 | self, base_params: dict[str, float]
624 | ) -> dict[str, float]:
625 | """Adjust parameters based on persona risk profile."""
626 | adjusted = {}
627 |
628 | for key, value in base_params.items():
629 | if "size" in key.lower() or "position" in key.lower():
630 | adjusted[key] = self.adjust_for_risk(value, "position_size")
631 | elif "stop" in key.lower():
632 | adjusted[key] = self.adjust_for_risk(value, "stop_loss")
633 | elif "target" in key.lower() or "profit" in key.lower():
634 | adjusted[key] = self.adjust_for_risk(value, "profit_target")
635 | else:
636 | adjusted[key] = value
637 |
638 | return adjusted
639 |
640 | def adjust_for_risk(self, value: float, parameter_type: str) -> float:
641 | """Adjust a value based on the persona's risk profile."""
642 | # Get average risk tolerance
643 | risk_avg = sum(self.persona.risk_tolerance) / 2
644 | risk_factor = risk_avg / 50 # Normalize to 1.0 at moderate risk
645 |
646 | # Adjust based on parameter type
647 | if parameter_type == "position_size":
648 | return min(value * risk_factor, self.persona.position_size_max)
649 | elif parameter_type == "stop_loss":
650 | return value * self.persona.stop_loss_multiplier
651 | elif parameter_type == "profit_target":
652 | return value * (2 - risk_factor) # Conservative = lower targets
653 | else:
654 | return value
655 |
```
--------------------------------------------------------------------------------
/maverick_mcp/api/routers/health_enhanced.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Comprehensive health check router for backtesting system.
3 |
4 | Provides detailed health monitoring including:
5 | - Component status (database, cache, external APIs)
6 | - Circuit breaker monitoring
7 | - Resource utilization
8 | - Readiness and liveness probes
9 | - Performance metrics
10 | """
11 |
12 | import asyncio
13 | import logging
14 | import time
15 | from datetime import UTC, datetime
16 | from pathlib import Path
17 | from typing import Any
18 |
19 | import psutil
20 | from fastapi import APIRouter, HTTPException
21 | from pydantic import BaseModel, Field
22 |
23 | from maverick_mcp.config.settings import get_settings
24 | from maverick_mcp.utils.circuit_breaker import get_circuit_breaker_status
25 |
26 | logger = logging.getLogger(__name__)
27 | settings = get_settings()
28 |
29 | router = APIRouter(prefix="/health", tags=["Health"])
30 |
31 | # Service start time for uptime calculation
32 | _start_time = time.time()
33 |
34 |
35 | class ComponentStatus(BaseModel):
36 | """Individual component health status."""
37 |
38 | name: str = Field(description="Component name")
39 | status: str = Field(description="Status (healthy/degraded/unhealthy)")
40 | response_time_ms: float | None = Field(description="Response time in milliseconds")
41 | last_check: str = Field(description="Timestamp of last health check")
42 | details: dict = Field(default_factory=dict, description="Additional status details")
43 | error: str | None = Field(default=None, description="Error message if unhealthy")
44 |
45 |
46 | class ResourceUsage(BaseModel):
47 | """System resource usage information."""
48 |
49 | cpu_percent: float = Field(description="CPU usage percentage")
50 | memory_percent: float = Field(description="Memory usage percentage")
51 | disk_percent: float = Field(description="Disk usage percentage")
52 | memory_used_mb: float = Field(description="Memory used in MB")
53 | memory_total_mb: float = Field(description="Total memory in MB")
54 | disk_used_gb: float = Field(description="Disk used in GB")
55 | disk_total_gb: float = Field(description="Total disk in GB")
56 | load_average: list[float] | None = Field(
57 | default=None, description="System load averages"
58 | )
59 |
60 |
61 | class CircuitBreakerStatus(BaseModel):
62 | """Circuit breaker status information."""
63 |
64 | name: str = Field(description="Circuit breaker name")
65 | state: str = Field(description="Current state (closed/open/half_open)")
66 | failure_count: int = Field(description="Current consecutive failure count")
67 | time_until_retry: float | None = Field(description="Seconds until retry allowed")
68 | metrics: dict = Field(description="Performance metrics")
69 |
70 |
71 | class DetailedHealthStatus(BaseModel):
72 | """Comprehensive health status with all components."""
73 |
74 | status: str = Field(
75 | description="Overall health status (healthy/degraded/unhealthy)"
76 | )
77 | timestamp: str = Field(description="Current timestamp")
78 | version: str = Field(description="Application version")
79 | uptime_seconds: float = Field(description="Service uptime in seconds")
80 | components: dict[str, ComponentStatus] = Field(
81 | description="Individual component statuses"
82 | )
83 | circuit_breakers: dict[str, CircuitBreakerStatus] = Field(
84 | description="Circuit breaker statuses"
85 | )
86 | resource_usage: ResourceUsage = Field(description="System resource usage")
87 | services: dict[str, str] = Field(description="External service statuses")
88 | checks_summary: dict[str, int] = Field(description="Summary of check results")
89 |
90 |
91 | class BasicHealthStatus(BaseModel):
92 | """Basic health status for simple health checks."""
93 |
94 | status: str = Field(
95 | description="Overall health status (healthy/degraded/unhealthy)"
96 | )
97 | timestamp: str = Field(description="Current timestamp")
98 | version: str = Field(description="Application version")
99 | uptime_seconds: float = Field(description="Service uptime in seconds")
100 |
101 |
102 | class ReadinessStatus(BaseModel):
103 | """Readiness probe status."""
104 |
105 | ready: bool = Field(description="Whether service is ready to accept traffic")
106 | timestamp: str = Field(description="Current timestamp")
107 | dependencies: dict[str, bool] = Field(description="Dependency readiness statuses")
108 | details: dict = Field(
109 | default_factory=dict, description="Additional readiness details"
110 | )
111 |
112 |
113 | class LivenessStatus(BaseModel):
114 | """Liveness probe status."""
115 |
116 | alive: bool = Field(description="Whether service is alive and functioning")
117 | timestamp: str = Field(description="Current timestamp")
118 | last_heartbeat: str = Field(description="Last heartbeat timestamp")
119 | details: dict = Field(
120 | default_factory=dict, description="Additional liveness details"
121 | )
122 |
123 |
124 | def _get_uptime_seconds() -> float:
125 | """Get service uptime in seconds."""
126 | return time.time() - _start_time
127 |
128 |
129 | def _get_resource_usage() -> ResourceUsage:
130 | """Get current system resource usage."""
131 | try:
132 | # CPU usage
133 | cpu_percent = psutil.cpu_percent(interval=1)
134 |
135 | # Memory usage
136 | memory = psutil.virtual_memory()
137 | memory_used_mb = (memory.total - memory.available) / (1024 * 1024)
138 | memory_total_mb = memory.total / (1024 * 1024)
139 |
140 | # Disk usage for current directory
141 | disk = psutil.disk_usage(Path.cwd())
142 | disk_used_gb = (disk.total - disk.free) / (1024 * 1024 * 1024)
143 | disk_total_gb = disk.total / (1024 * 1024 * 1024)
144 |
145 | # Load average (Unix systems only)
146 | load_average = None
147 | try:
148 | load_average = list(psutil.getloadavg())
149 | except (AttributeError, OSError):
150 | # Windows doesn't have load average
151 | pass
152 |
153 | return ResourceUsage(
154 | cpu_percent=round(cpu_percent, 2),
155 | memory_percent=round(memory.percent, 2),
156 | disk_percent=round(disk.percent, 2),
157 | memory_used_mb=round(memory_used_mb, 2),
158 | memory_total_mb=round(memory_total_mb, 2),
159 | disk_used_gb=round(disk_used_gb, 2),
160 | disk_total_gb=round(disk_total_gb, 2),
161 | load_average=load_average,
162 | )
163 | except Exception as e:
164 | logger.error(f"Failed to get resource usage: {e}")
165 | return ResourceUsage(
166 | cpu_percent=0.0,
167 | memory_percent=0.0,
168 | disk_percent=0.0,
169 | memory_used_mb=0.0,
170 | memory_total_mb=0.0,
171 | disk_used_gb=0.0,
172 | disk_total_gb=0.0,
173 | )
174 |
175 |
176 | async def _check_database_health() -> ComponentStatus:
177 | """Check database connectivity and health."""
178 | start_time = time.time()
179 | timestamp = datetime.now(UTC).isoformat()
180 |
181 | try:
182 | from maverick_mcp.data.models import get_db
183 |
184 | # Test database connection
185 | db_session = next(get_db())
186 | try:
187 | # Simple query to test connection
188 | result = db_session.execute("SELECT 1 as test")
189 | test_value = result.scalar()
190 |
191 | response_time_ms = (time.time() - start_time) * 1000
192 |
193 | if test_value == 1:
194 | return ComponentStatus(
195 | name="database",
196 | status="healthy",
197 | response_time_ms=round(response_time_ms, 2),
198 | last_check=timestamp,
199 | details={"connection": "active", "query_test": "passed"},
200 | )
201 | else:
202 | return ComponentStatus(
203 | name="database",
204 | status="unhealthy",
205 | response_time_ms=round(response_time_ms, 2),
206 | last_check=timestamp,
207 | error="Database query returned unexpected result",
208 | )
209 | finally:
210 | db_session.close()
211 |
212 | except Exception as e:
213 | response_time_ms = (time.time() - start_time) * 1000
214 | return ComponentStatus(
215 | name="database",
216 | status="unhealthy",
217 | response_time_ms=round(response_time_ms, 2),
218 | last_check=timestamp,
219 | error=str(e),
220 | )
221 |
222 |
223 | async def _check_cache_health() -> ComponentStatus:
224 | """Check Redis cache connectivity and health."""
225 | start_time = time.time()
226 | timestamp = datetime.now(UTC).isoformat()
227 |
228 | try:
229 | from maverick_mcp.data.cache import get_redis_client
230 |
231 | redis_client = get_redis_client()
232 | if redis_client is None:
233 | return ComponentStatus(
234 | name="cache",
235 | status="degraded",
236 | response_time_ms=0,
237 | last_check=timestamp,
238 | details={"type": "in_memory", "redis": "not_configured"},
239 | )
240 |
241 | # Test Redis connection
242 | await asyncio.to_thread(redis_client.ping)
243 | response_time_ms = (time.time() - start_time) * 1000
244 |
245 | # Get Redis info
246 | info = await asyncio.to_thread(redis_client.info)
247 |
248 | return ComponentStatus(
249 | name="cache",
250 | status="healthy",
251 | response_time_ms=round(response_time_ms, 2),
252 | last_check=timestamp,
253 | details={
254 | "type": "redis",
255 | "version": info.get("redis_version", "unknown"),
256 | "memory_usage": info.get("used_memory_human", "unknown"),
257 | "connected_clients": info.get("connected_clients", 0),
258 | },
259 | )
260 |
261 | except Exception as e:
262 | response_time_ms = (time.time() - start_time) * 1000
263 | return ComponentStatus(
264 | name="cache",
265 | status="degraded",
266 | response_time_ms=round(response_time_ms, 2),
267 | last_check=timestamp,
268 | details={"type": "fallback", "redis_error": str(e)},
269 | )
270 |
271 |
272 | async def _check_external_apis_health() -> dict[str, ComponentStatus]:
273 | """Check external API health using circuit breaker status."""
274 | timestamp = datetime.now(UTC).isoformat()
275 |
276 | # Map circuit breaker names to API names
277 | api_mapping = {
278 | "yfinance": "Yahoo Finance API",
279 | "finviz": "Finviz API",
280 | "fred_api": "FRED Economic Data API",
281 | "tiingo": "Tiingo Market Data API",
282 | "openrouter": "OpenRouter AI API",
283 | "exa": "Exa Search API",
284 | "news_api": "News API",
285 | "external_api": "External Services",
286 | }
287 |
288 | api_statuses = {}
289 | cb_status = get_circuit_breaker_status()
290 |
291 | for cb_name, display_name in api_mapping.items():
292 | cb_info = cb_status.get(cb_name)
293 |
294 | if cb_info:
295 | # Determine status based on circuit breaker state
296 | if cb_info["state"] == "closed":
297 | status = "healthy"
298 | error = None
299 | elif cb_info["state"] == "half_open":
300 | status = "degraded"
301 | error = "Circuit breaker testing recovery"
302 | else: # open
303 | status = "unhealthy"
304 | error = "Circuit breaker open due to failures"
305 |
306 | response_time = cb_info["metrics"].get("avg_response_time", 0)
307 |
308 | api_statuses[cb_name] = ComponentStatus(
309 | name=display_name,
310 | status=status,
311 | response_time_ms=round(response_time, 2) if response_time else None,
312 | last_check=timestamp,
313 | details={
314 | "circuit_breaker_state": cb_info["state"],
315 | "failure_count": cb_info["consecutive_failures"],
316 | "success_rate": cb_info["metrics"].get("success_rate", 0),
317 | },
318 | error=error,
319 | )
320 | else:
321 | # API not monitored by circuit breaker
322 | api_statuses[cb_name] = ComponentStatus(
323 | name=display_name,
324 | status="unknown",
325 | response_time_ms=None,
326 | last_check=timestamp,
327 | details={"monitoring": "not_configured"},
328 | )
329 |
330 | return api_statuses
331 |
332 |
333 | async def _check_ml_models_health() -> ComponentStatus:
334 | """Check ML model availability and health."""
335 | timestamp = datetime.now(UTC).isoformat()
336 |
337 | try:
338 | # Check if TA-Lib is available
339 | # Basic test of technical analysis libraries
340 | import numpy as np
341 |
342 | # Check if pandas-ta is available
343 | import pandas_ta as ta
344 | import talib
345 |
346 | test_data = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=float)
347 | sma_result = talib.SMA(test_data, timeperiod=5)
348 | sma_last_value = float(sma_result[-1])
349 |
350 | return ComponentStatus(
351 | name="ML Models & Libraries",
352 | status="healthy",
353 | response_time_ms=None,
354 | last_check=timestamp,
355 | details={
356 | "talib": f"available (v{getattr(talib, '__version__', 'unknown')})",
357 | "pandas_ta": f"available (v{getattr(ta, '__version__', 'unknown')})",
358 | "numpy": "available",
359 | "test_computation": "passed",
360 | "test_computation_sma_last": sma_last_value,
361 | },
362 | )
363 |
364 | except ImportError as e:
365 | return ComponentStatus(
366 | name="ML Models & Libraries",
367 | status="degraded",
368 | response_time_ms=None,
369 | last_check=timestamp,
370 | details={"missing_library": str(e)},
371 | error=f"Missing required library: {e}",
372 | )
373 | except Exception as e:
374 | return ComponentStatus(
375 | name="ML Models & Libraries",
376 | status="unhealthy",
377 | response_time_ms=None,
378 | last_check=timestamp,
379 | error=str(e),
380 | )
381 |
382 |
383 | async def _get_detailed_health_status() -> dict[str, Any]:
384 | """Get comprehensive health status for all components."""
385 | timestamp = datetime.now(UTC).isoformat()
386 |
387 | # Run all health checks concurrently
388 | db_task = _check_database_health()
389 | cache_task = _check_cache_health()
390 | apis_task = _check_external_apis_health()
391 | ml_task = _check_ml_models_health()
392 |
393 | try:
394 | db_status, cache_status, api_statuses, ml_status = await asyncio.gather(
395 | db_task, cache_task, apis_task, ml_task
396 | )
397 | except Exception as e:
398 | logger.error(f"Error running health checks: {e}")
399 | # Return minimal status on error
400 | return {
401 | "status": "unhealthy",
402 | "timestamp": timestamp,
403 | "version": getattr(settings, "version", "1.0.0"),
404 | "uptime_seconds": _get_uptime_seconds(),
405 | "components": {},
406 | "circuit_breakers": {},
407 | "resource_usage": _get_resource_usage(),
408 | "services": {},
409 | "checks_summary": {"healthy": 0, "degraded": 0, "unhealthy": 1},
410 | }
411 |
412 | # Combine all component statuses
413 | components = {
414 | "database": db_status,
415 | "cache": cache_status,
416 | "ml_models": ml_status,
417 | }
418 | components.update(api_statuses)
419 |
420 | # Get circuit breaker status
421 | cb_status = get_circuit_breaker_status()
422 | circuit_breakers = {}
423 | for name, status in cb_status.items():
424 | circuit_breakers[name] = CircuitBreakerStatus(
425 | name=status["name"],
426 | state=status["state"],
427 | failure_count=status["consecutive_failures"],
428 | time_until_retry=status["time_until_retry"],
429 | metrics=status["metrics"],
430 | )
431 |
432 | # Calculate overall health status
433 | healthy_count = sum(1 for c in components.values() if c.status == "healthy")
434 | degraded_count = sum(1 for c in components.values() if c.status == "degraded")
435 | unhealthy_count = sum(1 for c in components.values() if c.status == "unhealthy")
436 |
437 | if unhealthy_count > 0:
438 | overall_status = "unhealthy"
439 | elif degraded_count > 0:
440 | overall_status = "degraded"
441 | else:
442 | overall_status = "healthy"
443 |
444 | # Check service statuses based on circuit breakers
445 | services = {}
446 | for name, cb_info in cb_status.items():
447 | if cb_info["state"] == "open":
448 | services[name] = "down"
449 | elif cb_info["state"] == "half_open":
450 | services[name] = "degraded"
451 | else:
452 | services[name] = "up"
453 |
454 | return {
455 | "status": overall_status,
456 | "timestamp": timestamp,
457 | "version": getattr(settings, "version", "1.0.0"),
458 | "uptime_seconds": _get_uptime_seconds(),
459 | "components": components,
460 | "circuit_breakers": circuit_breakers,
461 | "resource_usage": _get_resource_usage(),
462 | "services": services,
463 | "checks_summary": {
464 | "healthy": healthy_count,
465 | "degraded": degraded_count,
466 | "unhealthy": unhealthy_count,
467 | },
468 | }
469 |
470 |
471 | @router.get("/", response_model=BasicHealthStatus)
472 | async def basic_health_check() -> BasicHealthStatus:
473 | """Basic health check endpoint.
474 |
475 | Returns simple health status without detailed component information.
476 | Suitable for basic monitoring and load balancer health checks.
477 | """
478 | try:
479 | # Get basic status from comprehensive health check
480 | detailed_status = await _get_detailed_health_status()
481 |
482 | return BasicHealthStatus(
483 | status=detailed_status["status"],
484 | timestamp=datetime.now(UTC).isoformat(),
485 | version=getattr(settings, "version", "1.0.0"),
486 | uptime_seconds=_get_uptime_seconds(),
487 | )
488 | except Exception as e:
489 | logger.error(f"Health check failed: {e}")
490 | return BasicHealthStatus(
491 | status="unhealthy",
492 | timestamp=datetime.now(UTC).isoformat(),
493 | version=getattr(settings, "version", "1.0.0"),
494 | uptime_seconds=_get_uptime_seconds(),
495 | )
496 |
497 |
498 | @router.get("/detailed", response_model=DetailedHealthStatus)
499 | async def detailed_health_check() -> DetailedHealthStatus:
500 | """Comprehensive health check with detailed component status.
501 |
502 | Returns detailed information about all system components including:
503 | - Database connectivity
504 | - Cache availability
505 | - External API status
506 | - Circuit breaker states
507 | - Resource utilization
508 | - ML model availability
509 |
510 | Returns:
511 | DetailedHealthStatus: Comprehensive health information
512 | """
513 | try:
514 | health_data = await _get_detailed_health_status()
515 | return DetailedHealthStatus(**health_data)
516 | except Exception as e:
517 | logger.error(f"Detailed health check failed: {e}")
518 | # Return minimal unhealthy status
519 | return DetailedHealthStatus(
520 | status="unhealthy",
521 | timestamp=datetime.now(UTC).isoformat(),
522 | version=getattr(settings, "version", "1.0.0"),
523 | uptime_seconds=_get_uptime_seconds(),
524 | components={},
525 | circuit_breakers={},
526 | resource_usage=ResourceUsage(
527 | cpu_percent=0.0,
528 | memory_percent=0.0,
529 | disk_percent=0.0,
530 | memory_used_mb=0.0,
531 | memory_total_mb=0.0,
532 | disk_used_gb=0.0,
533 | disk_total_gb=0.0,
534 | ),
535 | services={},
536 | checks_summary={"healthy": 0, "degraded": 0, "unhealthy": 1},
537 | )
538 |
539 |
540 | @router.get("/ready", response_model=ReadinessStatus)
541 | async def readiness_probe() -> ReadinessStatus:
542 | """Kubernetes-style readiness probe.
543 |
544 | Checks if the service is ready to accept traffic.
545 | Returns ready=true only if all critical dependencies are available.
546 | """
547 | try:
548 | health_data = await _get_detailed_health_status()
549 |
550 | # Critical dependencies for readiness
551 | critical_components = ["database"]
552 | dependencies = {}
553 |
554 | all_critical_ready = True
555 | for comp_name, comp_status in health_data["components"].items():
556 | if comp_name in critical_components:
557 | is_ready = comp_status.status in ["healthy", "degraded"]
558 | dependencies[comp_name] = is_ready
559 | if not is_ready:
560 | all_critical_ready = False
561 | else:
562 | # Non-critical components
563 | dependencies[comp_name] = comp_status.status != "unhealthy"
564 |
565 | return ReadinessStatus(
566 | ready=all_critical_ready,
567 | timestamp=datetime.now(UTC).isoformat(),
568 | dependencies=dependencies,
569 | details={
570 | "critical_components": critical_components,
571 | "overall_health": health_data["status"],
572 | },
573 | )
574 |
575 | except Exception as e:
576 | logger.error(f"Readiness probe failed: {e}")
577 | return ReadinessStatus(
578 | ready=False,
579 | timestamp=datetime.now(UTC).isoformat(),
580 | dependencies={},
581 | details={"error": str(e)},
582 | )
583 |
584 |
585 | @router.get("/live", response_model=LivenessStatus)
586 | async def liveness_probe() -> LivenessStatus:
587 | """Kubernetes-style liveness probe.
588 |
589 | Checks if the service is alive and functioning.
590 | Returns alive=true if the service can process basic requests.
591 | """
592 | try:
593 | # Simple check - if we can respond, we're alive
594 | current_time = datetime.now(UTC).isoformat()
595 |
596 | # Basic service functionality test
597 | uptime = _get_uptime_seconds()
598 |
599 | return LivenessStatus(
600 | alive=True,
601 | timestamp=current_time,
602 | last_heartbeat=current_time,
603 | details={
604 | "uptime_seconds": uptime,
605 | "service_name": settings.app_name,
606 | "process_id": psutil.Process().pid,
607 | },
608 | )
609 |
610 | except Exception as e:
611 | logger.error(f"Liveness probe failed: {e}")
612 | return LivenessStatus(
613 | alive=False,
614 | timestamp=datetime.now(UTC).isoformat(),
615 | last_heartbeat=datetime.now(UTC).isoformat(),
616 | details={"error": str(e)},
617 | )
618 |
619 |
620 | @router.get("/circuit-breakers", response_model=dict[str, CircuitBreakerStatus])
621 | async def get_circuit_breakers() -> dict[str, CircuitBreakerStatus]:
622 | """Get detailed circuit breaker status.
623 |
624 | Returns:
625 | Dictionary of circuit breaker statuses
626 | """
627 | cb_status = get_circuit_breaker_status()
628 |
629 | result = {}
630 | for name, status in cb_status.items():
631 | result[name] = CircuitBreakerStatus(
632 | name=status["name"],
633 | state=status["state"],
634 | failure_count=status["consecutive_failures"],
635 | time_until_retry=status["time_until_retry"],
636 | metrics=status["metrics"],
637 | )
638 |
639 | return result
640 |
641 |
642 | @router.post("/circuit-breakers/{name}/reset")
643 | async def reset_circuit_breaker(name: str) -> dict:
644 | """Reset a specific circuit breaker.
645 |
646 | Args:
647 | name: Circuit breaker name
648 |
649 | Returns:
650 | Success response
651 | """
652 | from maverick_mcp.utils.circuit_breaker import get_circuit_breaker
653 |
654 | breaker = get_circuit_breaker(name)
655 | if not breaker:
656 | raise HTTPException(
657 | status_code=404, detail=f"Circuit breaker '{name}' not found"
658 | )
659 |
660 | breaker.reset()
661 | logger.info(f"Circuit breaker '{name}' reset via API")
662 |
663 | return {"status": "success", "message": f"Circuit breaker '{name}' reset"}
664 |
665 |
666 | @router.post("/circuit-breakers/reset-all")
667 | async def reset_all_circuit_breakers() -> dict:
668 | """Reset all circuit breakers.
669 |
670 | Returns:
671 | Success response
672 | """
673 | from maverick_mcp.utils.circuit_breaker import reset_all_circuit_breakers
674 |
675 | reset_all_circuit_breakers()
676 | logger.info("All circuit breakers reset via API")
677 |
678 | return {"status": "success", "message": "All circuit breakers reset"}
679 |
```
--------------------------------------------------------------------------------
/maverick_mcp/backtesting/analysis.py:
--------------------------------------------------------------------------------
```python
1 | """Backtest result analysis utilities."""
2 |
3 | import logging
4 | from typing import Any
5 |
6 | import numpy as np
7 | import pandas as pd
8 | import vectorbt as vbt
9 |
10 | logger = logging.getLogger(__name__)
11 |
12 |
13 | def convert_to_native(value):
14 | """Convert numpy types to native Python types for JSON serialization."""
15 | if isinstance(value, np.int64 | np.int32 | np.int16 | np.int8):
16 | return int(value)
17 | elif isinstance(value, np.float64 | np.float32 | np.float16):
18 | return float(value)
19 | elif isinstance(value, np.ndarray):
20 | return value.tolist()
21 | elif hasattr(value, "item"): # For numpy scalars
22 | return value.item()
23 | elif pd.isna(value):
24 | return None
25 | return value
26 |
27 |
28 | class BacktestAnalyzer:
29 | """Analyzer for backtest results."""
30 |
31 | async def run_vectorbt_backtest(
32 | self,
33 | data: pd.DataFrame,
34 | entry_signals: pd.Series,
35 | exit_signals: pd.Series,
36 | initial_capital: float = 10000.0,
37 | fees: float = 0.001,
38 | slippage: float = 0.001,
39 | ) -> dict[str, Any]:
40 | """Run a backtest using VectorBT with given signals.
41 |
42 | Args:
43 | data: Price data with OHLCV columns
44 | entry_signals: Boolean series for entry signals
45 | exit_signals: Boolean series for exit signals
46 | initial_capital: Initial capital amount
47 | fees: Trading fees as percentage
48 | slippage: Slippage as percentage
49 |
50 | Returns:
51 | Backtest results dictionary
52 | """
53 | # Validate inputs to prevent empty array errors
54 | if data is None or len(data) == 0:
55 | logger.warning("Empty or invalid data provided to run_vectorbt_backtest")
56 | return self._create_empty_backtest_results(initial_capital)
57 |
58 | if entry_signals is None or exit_signals is None:
59 | logger.warning("Invalid signals provided to run_vectorbt_backtest")
60 | return self._create_empty_backtest_results(initial_capital)
61 |
62 | # Check for empty signals or all-False signals
63 | if (
64 | len(entry_signals) == 0
65 | or len(exit_signals) == 0
66 | or entry_signals.size == 0
67 | or exit_signals.size == 0
68 | ):
69 | logger.warning("Empty signal arrays provided to run_vectorbt_backtest")
70 | return self._create_empty_backtest_results(initial_capital)
71 |
72 | # Check if signals have any True values
73 | if not entry_signals.any() and not exit_signals.any():
74 | logger.info("No trading signals generated - returning buy-and-hold results")
75 | return self._create_buyhold_backtest_results(data, initial_capital)
76 |
77 | # Ensure we have close prices
78 | close = data["close"] if "close" in data.columns else data["Close"]
79 |
80 | try:
81 | # Run VectorBT portfolio simulation
82 | portfolio = vbt.Portfolio.from_signals(
83 | close=close,
84 | entries=entry_signals,
85 | exits=exit_signals,
86 | init_cash=initial_capital,
87 | fees=fees,
88 | slippage=slippage,
89 | freq="D",
90 | )
91 | except Exception as e:
92 | logger.error(f"VectorBT Portfolio.from_signals failed: {e}")
93 | return self._create_empty_backtest_results(initial_capital, error=str(e))
94 |
95 | # Extract metrics
96 | metrics = {
97 | "total_return": float(portfolio.total_return()),
98 | "annual_return": float(portfolio.annualized_return())
99 | if hasattr(portfolio, "annualized_return")
100 | else 0,
101 | "sharpe_ratio": float(portfolio.sharpe_ratio())
102 | if not np.isnan(portfolio.sharpe_ratio())
103 | else 0,
104 | "max_drawdown": float(portfolio.max_drawdown()),
105 | "win_rate": float(portfolio.trades.win_rate())
106 | if portfolio.trades.count() > 0
107 | else 0,
108 | "total_trades": int(portfolio.trades.count()),
109 | "profit_factor": float(portfolio.trades.profit_factor())
110 | if portfolio.trades.count() > 0
111 | else 0,
112 | }
113 |
114 | # Extract trades
115 | trades = []
116 | if portfolio.trades.count() > 0:
117 | try:
118 | # VectorBT trades are in a records array
119 | trade_records = portfolio.trades.records
120 | for i in range(len(trade_records)):
121 | trade = trade_records[i]
122 | trades.append(
123 | {
124 | "entry_time": convert_to_native(trade["entry_idx"])
125 | if "entry_idx" in trade.dtype.names
126 | else i,
127 | "exit_time": convert_to_native(trade["exit_idx"])
128 | if "exit_idx" in trade.dtype.names
129 | else i + 1,
130 | "pnl": convert_to_native(trade["pnl"])
131 | if "pnl" in trade.dtype.names
132 | else 0.0,
133 | "return": convert_to_native(trade["return"])
134 | if "return" in trade.dtype.names
135 | else 0.0,
136 | }
137 | )
138 | except (AttributeError, TypeError, KeyError) as e:
139 | # Fallback for different trade formats
140 | logger.debug(f"Could not extract detailed trades: {e}")
141 | trades = [
142 | {
143 | "total_trades": int(portfolio.trades.count()),
144 | "message": "Detailed trade data not available",
145 | }
146 | ]
147 |
148 | # Convert equity curve to ensure all values are Python native types
149 | equity_curve_raw = portfolio.value().to_dict()
150 | equity_curve = {
151 | str(k): convert_to_native(v) for k, v in equity_curve_raw.items()
152 | }
153 |
154 | # Also get drawdown series with proper conversion
155 | drawdown_raw = (
156 | portfolio.drawdown().to_dict() if hasattr(portfolio, "drawdown") else {}
157 | )
158 | drawdown_series = {
159 | str(k): convert_to_native(v) for k, v in drawdown_raw.items()
160 | }
161 |
162 | return {
163 | "metrics": metrics,
164 | "trades": trades,
165 | "equity_curve": equity_curve,
166 | "drawdown_series": drawdown_series,
167 | }
168 |
169 | def analyze(self, results: dict[str, Any]) -> dict[str, Any]:
170 | """Analyze backtest results and provide insights.
171 |
172 | Args:
173 | results: Backtest results from VectorBTEngine
174 |
175 | Returns:
176 | Analysis with performance grade, risk assessment, and recommendations
177 | """
178 | metrics = results.get("metrics", {})
179 | trades = results.get("trades", [])
180 |
181 | analysis = {
182 | "performance_grade": self._grade_performance(metrics),
183 | "risk_assessment": self._assess_risk(metrics),
184 | "trade_quality": self._analyze_trades(trades, metrics),
185 | "strengths": self._identify_strengths(metrics),
186 | "weaknesses": self._identify_weaknesses(metrics),
187 | "recommendations": self._generate_recommendations(metrics),
188 | "summary": self._generate_summary(metrics),
189 | }
190 |
191 | return analysis
192 |
193 | def _grade_performance(self, metrics: dict[str, float]) -> str:
194 | """Grade overall performance (A-F)."""
195 | score = 0
196 | max_score = 100
197 |
198 | # Sharpe ratio (30 points)
199 | sharpe = metrics.get("sharpe_ratio", 0)
200 | if sharpe >= 2.0:
201 | score += 30
202 | elif sharpe >= 1.5:
203 | score += 25
204 | elif sharpe >= 1.0:
205 | score += 20
206 | elif sharpe >= 0.5:
207 | score += 10
208 | else:
209 | score += 5
210 |
211 | # Total return (25 points)
212 | total_return = metrics.get("total_return", 0)
213 | if total_return >= 0.50: # 50%+
214 | score += 25
215 | elif total_return >= 0.30:
216 | score += 20
217 | elif total_return >= 0.15:
218 | score += 15
219 | elif total_return >= 0.05:
220 | score += 10
221 | elif total_return > 0:
222 | score += 5
223 |
224 | # Win rate (20 points)
225 | win_rate = metrics.get("win_rate", 0)
226 | if win_rate >= 0.60:
227 | score += 20
228 | elif win_rate >= 0.50:
229 | score += 15
230 | elif win_rate >= 0.40:
231 | score += 10
232 | else:
233 | score += 5
234 |
235 | # Max drawdown (15 points)
236 | max_dd = abs(metrics.get("max_drawdown", 0))
237 | if max_dd <= 0.10: # Less than 10%
238 | score += 15
239 | elif max_dd <= 0.20:
240 | score += 12
241 | elif max_dd <= 0.30:
242 | score += 8
243 | elif max_dd <= 0.40:
244 | score += 4
245 |
246 | # Profit factor (10 points)
247 | profit_factor = metrics.get("profit_factor", 0)
248 | if profit_factor >= 2.0:
249 | score += 10
250 | elif profit_factor >= 1.5:
251 | score += 8
252 | elif profit_factor >= 1.2:
253 | score += 5
254 | elif profit_factor > 1.0:
255 | score += 3
256 |
257 | # Convert score to grade
258 | percentage = (score / max_score) * 100
259 | if percentage >= 90:
260 | return "A"
261 | elif percentage >= 80:
262 | return "B"
263 | elif percentage >= 70:
264 | return "C"
265 | elif percentage >= 60:
266 | return "D"
267 | else:
268 | return "F"
269 |
270 | def _assess_risk(self, metrics: dict[str, float]) -> dict[str, Any]:
271 | """Assess risk characteristics."""
272 | max_dd = abs(metrics.get("max_drawdown", 0))
273 | sortino = metrics.get("sortino_ratio", 0)
274 | sharpe = metrics.get("sharpe_ratio", 0)
275 | calmar = metrics.get("calmar_ratio", 0)
276 | recovery = metrics.get("recovery_factor", 0)
277 |
278 | risk_level = "Low"
279 | if max_dd > 0.40:
280 | risk_level = "Very High"
281 | elif max_dd > 0.30:
282 | risk_level = "High"
283 | elif max_dd > 0.20:
284 | risk_level = "Medium"
285 | elif max_dd > 0.10:
286 | risk_level = "Low-Medium"
287 |
288 | return {
289 | "risk_level": risk_level,
290 | "max_drawdown": max_dd,
291 | "sortino_ratio": sortino,
292 | "calmar_ratio": calmar,
293 | "recovery_factor": recovery,
294 | "risk_adjusted_return": sortino if sortino > 0 else sharpe,
295 | "downside_protection": "Good"
296 | if sortino > 1.5
297 | else "Moderate"
298 | if sortino > 0.5
299 | else "Poor",
300 | }
301 |
302 | def _analyze_trades(
303 | self, trades: list[dict], metrics: dict[str, float]
304 | ) -> dict[str, Any]:
305 | """Analyze trade quality and patterns."""
306 | if not trades:
307 | return {
308 | "quality": "No trades",
309 | "total_trades": 0,
310 | "frequency": "None",
311 | }
312 |
313 | total_trades = metrics.get("total_trades", 0)
314 | win_rate = metrics.get("win_rate", 0)
315 | avg_duration = metrics.get("avg_duration", 0)
316 |
317 | # Determine trade frequency
318 | if total_trades < 10:
319 | frequency = "Very Low"
320 | elif total_trades < 50:
321 | frequency = "Low"
322 | elif total_trades < 100:
323 | frequency = "Moderate"
324 | elif total_trades < 200:
325 | frequency = "High"
326 | else:
327 | frequency = "Very High"
328 |
329 | # Determine trade quality
330 | if win_rate >= 0.60 and metrics.get("profit_factor", 0) >= 1.5:
331 | quality = "Excellent"
332 | elif win_rate >= 0.50 and metrics.get("profit_factor", 0) >= 1.2:
333 | quality = "Good"
334 | elif win_rate >= 0.40:
335 | quality = "Average"
336 | else:
337 | quality = "Poor"
338 |
339 | return {
340 | "quality": quality,
341 | "total_trades": total_trades,
342 | "frequency": frequency,
343 | "win_rate": win_rate,
344 | "avg_win": metrics.get("avg_win", 0),
345 | "avg_loss": metrics.get("avg_loss", 0),
346 | "best_trade": metrics.get("best_trade", 0),
347 | "worst_trade": metrics.get("worst_trade", 0),
348 | "avg_duration_days": avg_duration,
349 | "risk_reward_ratio": metrics.get("risk_reward_ratio", 0),
350 | }
351 |
352 | def _identify_strengths(self, metrics: dict[str, float]) -> list[str]:
353 | """Identify strategy strengths."""
354 | strengths = []
355 |
356 | if metrics.get("sharpe_ratio", 0) >= 1.5:
357 | strengths.append("Excellent risk-adjusted returns")
358 | if metrics.get("win_rate", 0) >= 0.60:
359 | strengths.append("High win rate")
360 | if abs(metrics.get("max_drawdown", 0)) <= 0.15:
361 | strengths.append("Low maximum drawdown")
362 | if metrics.get("profit_factor", 0) >= 1.5:
363 | strengths.append("Strong profit factor")
364 | if metrics.get("sortino_ratio", 0) >= 2.0:
365 | strengths.append("Excellent downside protection")
366 | if metrics.get("calmar_ratio", 0) >= 1.0:
367 | strengths.append("Good return vs drawdown ratio")
368 | if metrics.get("recovery_factor", 0) >= 3.0:
369 | strengths.append("Quick drawdown recovery")
370 | if metrics.get("total_return", 0) >= 0.30:
371 | strengths.append("High total returns")
372 |
373 | return strengths if strengths else ["Consistent performance"]
374 |
375 | def _identify_weaknesses(self, metrics: dict[str, float]) -> list[str]:
376 | """Identify strategy weaknesses."""
377 | weaknesses = []
378 |
379 | if metrics.get("sharpe_ratio", 0) < 0.5:
380 | weaknesses.append("Poor risk-adjusted returns")
381 | if metrics.get("win_rate", 0) < 0.40:
382 | weaknesses.append("Low win rate")
383 | if abs(metrics.get("max_drawdown", 0)) > 0.30:
384 | weaknesses.append("High maximum drawdown")
385 | if metrics.get("profit_factor", 0) < 1.0:
386 | weaknesses.append("Unprofitable trades overall")
387 | if metrics.get("total_trades", 0) < 10:
388 | weaknesses.append("Insufficient trade signals")
389 | if metrics.get("sortino_ratio", 0) < 0:
390 | weaknesses.append("Poor downside protection")
391 | if metrics.get("total_return", 0) < 0:
392 | weaknesses.append("Negative returns")
393 |
394 | return weaknesses if weaknesses else ["Room for optimization"]
395 |
396 | def _generate_recommendations(self, metrics: dict[str, float]) -> list[str]:
397 | """Generate improvement recommendations."""
398 | recommendations = []
399 |
400 | # Risk management recommendations
401 | if abs(metrics.get("max_drawdown", 0)) > 0.25:
402 | recommendations.append(
403 | "Implement tighter stop-loss rules to reduce drawdowns"
404 | )
405 |
406 | # Win rate improvements
407 | if metrics.get("win_rate", 0) < 0.45:
408 | recommendations.append("Refine entry signals to improve win rate")
409 |
410 | # Trade frequency
411 | if metrics.get("total_trades", 0) < 20:
412 | recommendations.append(
413 | "Consider more sensitive parameters for increased signals"
414 | )
415 | elif metrics.get("total_trades", 0) > 200:
416 | recommendations.append("Filter signals to reduce overtrading")
417 |
418 | # Risk-reward optimization
419 | if metrics.get("risk_reward_ratio", 0) < 1.5:
420 | recommendations.append("Adjust exit strategy for better risk-reward ratio")
421 |
422 | # Profit factor improvements
423 | if metrics.get("profit_factor", 0) < 1.2:
424 | recommendations.append(
425 | "Focus on cutting losses quicker and letting winners run"
426 | )
427 |
428 | # Sharpe ratio improvements
429 | if metrics.get("sharpe_ratio", 0) < 1.0:
430 | recommendations.append("Consider position sizing based on volatility")
431 |
432 | # Kelly criterion
433 | kelly = metrics.get("kelly_criterion", 0)
434 | if kelly > 0 and kelly < 0.25:
435 | recommendations.append(
436 | f"Consider position size of {kelly * 100:.1f}% based on Kelly Criterion"
437 | )
438 |
439 | return (
440 | recommendations
441 | if recommendations
442 | else ["Strategy performing well, consider live testing"]
443 | )
444 |
445 | def _generate_summary(self, metrics: dict[str, float]) -> str:
446 | """Generate a text summary of the backtest."""
447 | total_return = metrics.get("total_return", 0) * 100
448 | sharpe = metrics.get("sharpe_ratio", 0)
449 | max_dd = abs(metrics.get("max_drawdown", 0)) * 100
450 | win_rate = metrics.get("win_rate", 0) * 100
451 | total_trades = metrics.get("total_trades", 0)
452 |
453 | summary = f"The strategy generated a {total_return:.1f}% return with a Sharpe ratio of {sharpe:.2f}. "
454 | summary += f"Maximum drawdown was {max_dd:.1f}% with a {win_rate:.1f}% win rate across {total_trades} trades. "
455 |
456 | if sharpe >= 1.5 and max_dd <= 20:
457 | summary += (
458 | "Overall performance is excellent with strong risk-adjusted returns."
459 | )
460 | elif sharpe >= 1.0 and max_dd <= 30:
461 | summary += "Performance is good with acceptable risk levels."
462 | elif sharpe >= 0.5:
463 | summary += "Performance is moderate and could benefit from optimization."
464 | else:
465 | summary += "Performance needs significant improvement before live trading."
466 |
467 | return summary
468 |
469 | def compare_strategies(self, results_list: list[dict[str, Any]]) -> dict[str, Any]:
470 | """Compare multiple strategy results.
471 |
472 | Args:
473 | results_list: List of backtest results to compare
474 |
475 | Returns:
476 | Comparison analysis with rankings
477 | """
478 | if not results_list:
479 | return {"error": "No results to compare"}
480 |
481 | comparisons = []
482 |
483 | for result in results_list:
484 | metrics = result.get("metrics", {})
485 | comparisons.append(
486 | {
487 | "strategy": result.get("strategy", "Unknown"),
488 | "parameters": result.get("parameters", {}),
489 | "total_return": metrics.get("total_return", 0),
490 | "sharpe_ratio": metrics.get("sharpe_ratio", 0),
491 | "max_drawdown": abs(metrics.get("max_drawdown", 0)),
492 | "win_rate": metrics.get("win_rate", 0),
493 | "profit_factor": metrics.get("profit_factor", 0),
494 | "total_trades": metrics.get("total_trades", 0),
495 | "grade": self._grade_performance(metrics),
496 | }
497 | )
498 |
499 | # Sort by Sharpe ratio as default ranking
500 | comparisons.sort(key=lambda x: x["sharpe_ratio"], reverse=True)
501 |
502 | # Add rankings
503 | for i, comp in enumerate(comparisons, 1):
504 | comp["rank"] = i
505 |
506 | # Find best in each category
507 | best_return = max(comparisons, key=lambda x: x["total_return"])
508 | best_sharpe = max(comparisons, key=lambda x: x["sharpe_ratio"])
509 | best_drawdown = min(comparisons, key=lambda x: x["max_drawdown"])
510 | best_win_rate = max(comparisons, key=lambda x: x["win_rate"])
511 |
512 | return {
513 | "rankings": comparisons,
514 | "best_overall": comparisons[0] if comparisons else None,
515 | "best_return": best_return,
516 | "best_sharpe": best_sharpe,
517 | "best_drawdown": best_drawdown,
518 | "best_win_rate": best_win_rate,
519 | "summary": self._generate_comparison_summary(comparisons),
520 | }
521 |
522 | def _generate_comparison_summary(self, comparisons: list[dict]) -> str:
523 | """Generate summary of strategy comparison."""
524 | if not comparisons:
525 | return "No strategies to compare"
526 |
527 | best = comparisons[0]
528 | summary = f"The best performing strategy is {best['strategy']} "
529 | summary += f"with a Sharpe ratio of {best['sharpe_ratio']:.2f} "
530 | summary += f"and total return of {best['total_return'] * 100:.1f}%. "
531 |
532 | if len(comparisons) > 1:
533 | summary += (
534 | f"It outperformed {len(comparisons) - 1} other strategies tested."
535 | )
536 |
537 | return summary
538 |
539 | def _create_empty_backtest_results(
540 | self, initial_capital: float, error: str = None
541 | ) -> dict[str, Any]:
542 | """Create empty backtest results when no valid signals are available.
543 |
544 | Args:
545 | initial_capital: Initial capital amount
546 | error: Optional error message to include
547 |
548 | Returns:
549 | Empty backtest results dictionary
550 | """
551 | return {
552 | "metrics": {
553 | "total_return": 0.0,
554 | "annual_return": 0.0,
555 | "sharpe_ratio": 0.0,
556 | "max_drawdown": 0.0,
557 | "win_rate": 0.0,
558 | "total_trades": 0,
559 | "profit_factor": 0.0,
560 | },
561 | "trades": [],
562 | "equity_curve": {str(0): initial_capital},
563 | "drawdown_series": {str(0): 0.0},
564 | "error": error,
565 | "message": "No trading signals generated - empty backtest results returned",
566 | }
567 |
568 | def _create_buyhold_backtest_results(
569 | self, data: pd.DataFrame, initial_capital: float
570 | ) -> dict[str, Any]:
571 | """Create buy-and-hold backtest results when no trading signals are available.
572 |
573 | Args:
574 | data: Price data
575 | initial_capital: Initial capital amount
576 |
577 | Returns:
578 | Buy-and-hold backtest results dictionary
579 | """
580 | try:
581 | # Calculate buy and hold performance
582 | close = data["close"] if "close" in data.columns else data["Close"]
583 | if len(close) == 0:
584 | return self._create_empty_backtest_results(initial_capital)
585 |
586 | start_price = close.iloc[0]
587 | end_price = close.iloc[-1]
588 | total_return = (end_price - start_price) / start_price
589 |
590 | # Simple buy and hold equity curve
591 | normalized_prices = close / start_price * initial_capital
592 | equity_curve = {
593 | str(idx): convert_to_native(val)
594 | for idx, val in normalized_prices.to_dict().items()
595 | }
596 |
597 | # Calculate drawdown for buy and hold
598 | cummax = normalized_prices.expanding().max()
599 | drawdown = (normalized_prices - cummax) / cummax
600 | drawdown_series = {
601 | str(idx): convert_to_native(val)
602 | for idx, val in drawdown.to_dict().items()
603 | }
604 |
605 | return {
606 | "metrics": {
607 | "total_return": float(total_return),
608 | "annual_return": float(total_return * 252 / len(data))
609 | if len(data) > 0
610 | else 0.0,
611 | "sharpe_ratio": 0.0, # Cannot calculate without trading
612 | "max_drawdown": float(drawdown.min()) if len(drawdown) > 0 else 0.0,
613 | "win_rate": 0.0, # No trades
614 | "total_trades": 0,
615 | "profit_factor": 0.0, # No trades
616 | },
617 | "trades": [],
618 | "equity_curve": equity_curve,
619 | "drawdown_series": drawdown_series,
620 | "message": "No trading signals generated - returning buy-and-hold performance",
621 | }
622 | except Exception as e:
623 | logger.error(f"Error creating buy-and-hold results: {e}")
624 | return self._create_empty_backtest_results(initial_capital, error=str(e))
625 |
```
--------------------------------------------------------------------------------
/maverick_mcp/data/validation.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Data Quality Validation Module for MaverickMCP.
3 |
4 | This module provides comprehensive data validation functionality for
5 | stock price data, backtesting data, and general data quality checks.
6 | Ensures data integrity before processing and backtesting operations.
7 | """
8 |
9 | import logging
10 | from datetime import date, datetime
11 | from typing import Any
12 |
13 | import numpy as np
14 | import pandas as pd
15 | from pandas import DataFrame
16 |
17 | from maverick_mcp.exceptions import ValidationError
18 |
19 | logger = logging.getLogger(__name__)
20 |
21 |
22 | class DataValidator:
23 | """Comprehensive data validation for stock market and backtesting data."""
24 |
25 | @staticmethod
26 | def validate_date_range(
27 | start_date: str | datetime | date,
28 | end_date: str | datetime | date,
29 | allow_future: bool = False,
30 | max_range_days: int | None = None,
31 | ) -> tuple[datetime, datetime]:
32 | """
33 | Validate date range for data queries.
34 |
35 | Args:
36 | start_date: Start date for the range
37 | end_date: End date for the range
38 | allow_future: Whether to allow future dates
39 | max_range_days: Maximum allowed days in range
40 |
41 | Returns:
42 | Tuple of validated (start_date, end_date) as datetime objects
43 |
44 | Raises:
45 | ValidationError: If dates are invalid
46 | """
47 | # Convert to datetime objects
48 | if isinstance(start_date, str):
49 | try:
50 | start_dt = pd.to_datetime(start_date).to_pydatetime()
51 | except Exception as e:
52 | raise ValidationError(f"Invalid start_date format: {start_date}") from e
53 | elif isinstance(start_date, date):
54 | start_dt = datetime.combine(start_date, datetime.min.time())
55 | else:
56 | start_dt = start_date
57 |
58 | if isinstance(end_date, str):
59 | try:
60 | end_dt = pd.to_datetime(end_date).to_pydatetime()
61 | except Exception as e:
62 | raise ValidationError(f"Invalid end_date format: {end_date}") from e
63 | elif isinstance(end_date, date):
64 | end_dt = datetime.combine(end_date, datetime.min.time())
65 | else:
66 | end_dt = end_date
67 |
68 | # Validate chronological order
69 | if start_dt > end_dt:
70 | raise ValidationError(
71 | f"Start date {start_dt.date()} must be before end date {end_dt.date()}"
72 | )
73 |
74 | # Check future dates if not allowed
75 | if not allow_future:
76 | today = datetime.now().date()
77 | if start_dt.date() > today:
78 | raise ValidationError(
79 | f"Start date {start_dt.date()} cannot be in the future"
80 | )
81 | if end_dt.date() > today:
82 | logger.warning(
83 | f"End date {end_dt.date()} is in the future, using today instead"
84 | )
85 | end_dt = datetime.combine(today, datetime.min.time())
86 |
87 | # Check maximum range
88 | if max_range_days:
89 | range_days = (end_dt - start_dt).days
90 | if range_days > max_range_days:
91 | raise ValidationError(
92 | f"Date range too large: {range_days} days (max: {max_range_days} days)"
93 | )
94 |
95 | return start_dt, end_dt
96 |
97 | @staticmethod
98 | def validate_data_quality(
99 | data: DataFrame,
100 | required_columns: list[str] | None = None,
101 | min_rows: int = 1,
102 | max_missing_ratio: float = 0.1,
103 | check_duplicates: bool = True,
104 | ) -> dict[str, Any]:
105 | """
106 | Validate general data quality of a DataFrame.
107 |
108 | Args:
109 | data: DataFrame to validate
110 | required_columns: List of required columns
111 | min_rows: Minimum number of rows required
112 | max_missing_ratio: Maximum ratio of missing values allowed
113 | check_duplicates: Whether to check for duplicate rows
114 |
115 | Returns:
116 | Dictionary with validation results and quality metrics
117 |
118 | Raises:
119 | ValidationError: If validation fails
120 | """
121 | if data is None or data.empty:
122 | raise ValidationError("Data is None or empty")
123 |
124 | validation_results = {
125 | "passed": True,
126 | "warnings": [],
127 | "errors": [],
128 | "metrics": {
129 | "total_rows": len(data),
130 | "total_columns": len(data.columns),
131 | "missing_values": data.isnull().sum().sum(),
132 | "duplicate_rows": 0,
133 | },
134 | }
135 |
136 | # Check minimum rows
137 | if len(data) < min_rows:
138 | error_msg = f"Insufficient data: {len(data)} rows (minimum: {min_rows})"
139 | validation_results["errors"].append(error_msg)
140 | validation_results["passed"] = False
141 |
142 | # Check required columns
143 | if required_columns:
144 | missing_cols = set(required_columns) - set(data.columns)
145 | if missing_cols:
146 | error_msg = f"Missing required columns: {list(missing_cols)}"
147 | validation_results["errors"].append(error_msg)
148 | validation_results["passed"] = False
149 |
150 | # Check missing values ratio
151 | total_cells = len(data) * len(data.columns)
152 | if total_cells > 0:
153 | missing_ratio = (
154 | validation_results["metrics"]["missing_values"] / total_cells
155 | )
156 | validation_results["metrics"]["missing_ratio"] = missing_ratio
157 |
158 | if missing_ratio > max_missing_ratio:
159 | error_msg = f"Too many missing values: {missing_ratio:.2%} (max: {max_missing_ratio:.2%})"
160 | validation_results["errors"].append(error_msg)
161 | validation_results["passed"] = False
162 |
163 | # Check for duplicate rows
164 | if check_duplicates:
165 | duplicate_count = data.duplicated().sum()
166 | validation_results["metrics"]["duplicate_rows"] = duplicate_count
167 |
168 | if duplicate_count > 0:
169 | warning_msg = f"Found {duplicate_count} duplicate rows"
170 | validation_results["warnings"].append(warning_msg)
171 |
172 | # Check for completely empty columns
173 | empty_columns = data.columns[data.isnull().all()].tolist()
174 | if empty_columns:
175 | warning_msg = f"Completely empty columns: {empty_columns}"
176 | validation_results["warnings"].append(warning_msg)
177 |
178 | return validation_results
179 |
180 | @staticmethod
181 | def validate_price_data(
182 | data: DataFrame, symbol: str = "Unknown", strict_mode: bool = True
183 | ) -> dict[str, Any]:
184 | """
185 | Validate OHLCV stock price data integrity.
186 |
187 | Args:
188 | data: DataFrame with OHLCV data
189 | symbol: Stock symbol for error messages
190 | strict_mode: Whether to apply strict validation rules
191 |
192 | Returns:
193 | Dictionary with validation results and metrics
194 |
195 | Raises:
196 | ValidationError: If validation fails in strict mode
197 | """
198 | expected_columns = ["open", "high", "low", "close"]
199 |
200 | # Basic data quality check
201 | quality_results = DataValidator.validate_data_quality(
202 | data,
203 | required_columns=expected_columns,
204 | min_rows=1,
205 | max_missing_ratio=0.05, # Allow 5% missing values for price data
206 | )
207 |
208 | validation_results = {
209 | "passed": quality_results["passed"],
210 | "warnings": quality_results["warnings"].copy(),
211 | "errors": quality_results["errors"].copy(),
212 | "metrics": quality_results["metrics"].copy(),
213 | "symbol": symbol,
214 | "price_validation": {
215 | "negative_prices": 0,
216 | "zero_prices": 0,
217 | "invalid_ohlc_relationships": 0,
218 | "extreme_price_changes": 0,
219 | "volume_anomalies": 0,
220 | },
221 | }
222 |
223 | if data.empty:
224 | return validation_results
225 |
226 | # Check for negative prices
227 | price_cols = [col for col in expected_columns if col in data.columns]
228 | for col in price_cols:
229 | if col in data.columns:
230 | negative_count = (data[col] < 0).sum()
231 | if negative_count > 0:
232 | error_msg = (
233 | f"Found {negative_count} negative {col} prices for {symbol}"
234 | )
235 | validation_results["errors"].append(error_msg)
236 | validation_results["price_validation"]["negative_prices"] += (
237 | negative_count
238 | )
239 | validation_results["passed"] = False
240 |
241 | # Check for zero prices
242 | for col in price_cols:
243 | if col in data.columns:
244 | zero_count = (data[col] == 0).sum()
245 | if zero_count > 0:
246 | warning_msg = f"Found {zero_count} zero {col} prices for {symbol}"
247 | validation_results["warnings"].append(warning_msg)
248 | validation_results["price_validation"]["zero_prices"] += zero_count
249 |
250 | # Validate OHLC relationships (High >= Open, Close, Low; Low <= Open, Close)
251 | if all(col in data.columns for col in ["open", "high", "low", "close"]):
252 | # High should be >= Open, Low, Close
253 | high_violations = (
254 | (data["high"] < data["open"])
255 | | (data["high"] < data["low"])
256 | | (data["high"] < data["close"])
257 | ).sum()
258 |
259 | # Low should be <= Open, High, Close
260 | low_violations = (
261 | (data["low"] > data["open"])
262 | | (data["low"] > data["high"])
263 | | (data["low"] > data["close"])
264 | ).sum()
265 |
266 | total_ohlc_violations = high_violations + low_violations
267 | if total_ohlc_violations > 0:
268 | error_msg = f"OHLC relationship violations for {symbol}: {total_ohlc_violations} bars"
269 | validation_results["errors"].append(error_msg)
270 | validation_results["price_validation"]["invalid_ohlc_relationships"] = (
271 | total_ohlc_violations
272 | )
273 | validation_results["passed"] = False
274 |
275 | # Check for extreme price changes (>50% daily moves)
276 | if "close" in data.columns and len(data) > 1:
277 | daily_returns = data["close"].pct_change().dropna()
278 | extreme_changes = (daily_returns.abs() > 0.5).sum()
279 | if extreme_changes > 0:
280 | warning_msg = (
281 | f"Found {extreme_changes} extreme price changes (>50%) for {symbol}"
282 | )
283 | validation_results["warnings"].append(warning_msg)
284 | validation_results["price_validation"]["extreme_price_changes"] = (
285 | extreme_changes
286 | )
287 |
288 | # Validate volume data if present
289 | if "volume" in data.columns:
290 | negative_volume = (data["volume"] < 0).sum()
291 | if negative_volume > 0:
292 | error_msg = (
293 | f"Found {negative_volume} negative volume values for {symbol}"
294 | )
295 | validation_results["errors"].append(error_msg)
296 | validation_results["price_validation"]["volume_anomalies"] += (
297 | negative_volume
298 | )
299 | validation_results["passed"] = False
300 |
301 | # Check for suspiciously high volume (>10x median)
302 | if len(data) > 10:
303 | median_volume = data["volume"].median()
304 | if median_volume > 0:
305 | high_volume_count = (data["volume"] > median_volume * 10).sum()
306 | if high_volume_count > 0:
307 | validation_results["price_validation"]["volume_anomalies"] += (
308 | high_volume_count
309 | )
310 |
311 | # Check data continuity (gaps in date index)
312 | if hasattr(data.index, "to_series"):
313 | date_diffs = data.index.to_series().diff()[1:]
314 | if len(date_diffs) > 0:
315 | # Check for gaps larger than 7 days (weekend + holiday)
316 | large_gaps = (date_diffs > pd.Timedelta(days=7)).sum()
317 | if large_gaps > 0:
318 | warning_msg = f"Found {large_gaps} large time gaps (>7 days) in data for {symbol}"
319 | validation_results["warnings"].append(warning_msg)
320 |
321 | # Raise error in strict mode if validation failed
322 | if strict_mode and not validation_results["passed"]:
323 | error_summary = "; ".join(validation_results["errors"])
324 | raise ValidationError(
325 | f"Price data validation failed for {symbol}: {error_summary}"
326 | )
327 |
328 | return validation_results
329 |
330 | @staticmethod
331 | def validate_batch_data(
332 | batch_data: dict[str, DataFrame],
333 | min_symbols: int = 1,
334 | max_symbols: int = 100,
335 | validate_individual: bool = True,
336 | ) -> dict[str, Any]:
337 | """
338 | Validate batch data containing multiple symbol DataFrames.
339 |
340 | Args:
341 | batch_data: Dictionary mapping symbols to DataFrames
342 | min_symbols: Minimum number of symbols required
343 | max_symbols: Maximum number of symbols allowed
344 | validate_individual: Whether to validate each symbol's data
345 |
346 | Returns:
347 | Dictionary with batch validation results
348 |
349 | Raises:
350 | ValidationError: If batch validation fails
351 | """
352 | if not isinstance(batch_data, dict):
353 | raise ValidationError("Batch data must be a dictionary")
354 |
355 | validation_results = {
356 | "passed": True,
357 | "warnings": [],
358 | "errors": [],
359 | "metrics": {
360 | "total_symbols": len(batch_data),
361 | "valid_symbols": 0,
362 | "invalid_symbols": 0,
363 | "empty_symbols": 0,
364 | "total_rows": 0,
365 | },
366 | "symbol_results": {},
367 | }
368 |
369 | # Check symbol count
370 | symbol_count = len(batch_data)
371 | if symbol_count < min_symbols:
372 | error_msg = f"Insufficient symbols: {symbol_count} (minimum: {min_symbols})"
373 | validation_results["errors"].append(error_msg)
374 | validation_results["passed"] = False
375 |
376 | if symbol_count > max_symbols:
377 | error_msg = f"Too many symbols: {symbol_count} (maximum: {max_symbols})"
378 | validation_results["errors"].append(error_msg)
379 | validation_results["passed"] = False
380 |
381 | # Validate each symbol's data
382 | for symbol, data in batch_data.items():
383 | try:
384 | if data is None or data.empty:
385 | validation_results["metrics"]["empty_symbols"] += 1
386 | validation_results["symbol_results"][symbol] = {
387 | "passed": False,
388 | "error": "Empty or None data",
389 | }
390 | continue
391 |
392 | if validate_individual:
393 | # Validate price data for each symbol
394 | symbol_validation = DataValidator.validate_price_data(
395 | data, symbol, strict_mode=False
396 | )
397 | validation_results["symbol_results"][symbol] = symbol_validation
398 |
399 | if symbol_validation["passed"]:
400 | validation_results["metrics"]["valid_symbols"] += 1
401 | else:
402 | validation_results["metrics"]["invalid_symbols"] += 1
403 | # Aggregate errors
404 | for error in symbol_validation["errors"]:
405 | validation_results["errors"].append(f"{symbol}: {error}")
406 |
407 | # Don't fail entire batch for individual symbol issues
408 | # validation_results["passed"] = False
409 | else:
410 | validation_results["metrics"]["valid_symbols"] += 1
411 | validation_results["symbol_results"][symbol] = {
412 | "passed": True,
413 | "rows": len(data),
414 | }
415 |
416 | validation_results["metrics"]["total_rows"] += len(data)
417 |
418 | except Exception as e:
419 | validation_results["metrics"]["invalid_symbols"] += 1
420 | validation_results["symbol_results"][symbol] = {
421 | "passed": False,
422 | "error": str(e),
423 | }
424 | validation_results["errors"].append(f"{symbol}: Validation error - {e}")
425 |
426 | # Summary metrics
427 | validation_results["metrics"]["success_rate"] = (
428 | validation_results["metrics"]["valid_symbols"] / symbol_count
429 | if symbol_count > 0
430 | else 0.0
431 | )
432 |
433 | # Add warnings for low success rate
434 | if validation_results["metrics"]["success_rate"] < 0.8:
435 | warning_msg = (
436 | f"Low success rate: {validation_results['metrics']['success_rate']:.1%}"
437 | )
438 | validation_results["warnings"].append(warning_msg)
439 |
440 | return validation_results
441 |
442 | @staticmethod
443 | def validate_technical_indicators(
444 | data: DataFrame, indicators: dict[str, Any], symbol: str = "Unknown"
445 | ) -> dict[str, Any]:
446 | """
447 | Validate technical indicator data.
448 |
449 | Args:
450 | data: DataFrame with technical indicator data
451 | indicators: Dictionary of indicator configurations
452 | symbol: Symbol name for error messages
453 |
454 | Returns:
455 | Dictionary with validation results
456 | """
457 | validation_results = {
458 | "passed": True,
459 | "warnings": [],
460 | "errors": [],
461 | "metrics": {
462 | "total_indicators": len(indicators),
463 | "valid_indicators": 0,
464 | "nan_counts": {},
465 | },
466 | }
467 |
468 | for indicator_name, _config in indicators.items():
469 | if indicator_name not in data.columns:
470 | error_msg = f"Missing indicator '{indicator_name}' for {symbol}"
471 | validation_results["errors"].append(error_msg)
472 | validation_results["passed"] = False
473 | continue
474 |
475 | indicator_data = data[indicator_name]
476 |
477 | # Count NaN values
478 | nan_count = indicator_data.isnull().sum()
479 | validation_results["metrics"]["nan_counts"][indicator_name] = nan_count
480 |
481 | # Check for excessive NaN values
482 | if len(data) > 0:
483 | nan_ratio = nan_count / len(data)
484 | if nan_ratio > 0.5: # More than 50% NaN
485 | warning_msg = (
486 | f"High NaN ratio for '{indicator_name}': {nan_ratio:.1%}"
487 | )
488 | validation_results["warnings"].append(warning_msg)
489 | elif nan_ratio == 0:
490 | validation_results["metrics"]["valid_indicators"] += 1
491 |
492 | # Check for infinite values
493 | if np.any(np.isinf(indicator_data.fillna(0))):
494 | error_msg = f"Infinite values found in '{indicator_name}' for {symbol}"
495 | validation_results["errors"].append(error_msg)
496 | validation_results["passed"] = False
497 |
498 | return validation_results
499 |
500 | @classmethod
501 | def create_validation_report(
502 | cls, validation_results: dict[str, Any], include_warnings: bool = True
503 | ) -> str:
504 | """
505 | Create a human-readable validation report.
506 |
507 | Args:
508 | validation_results: Results from validation methods
509 | include_warnings: Whether to include warnings in report
510 |
511 | Returns:
512 | Formatted validation report string
513 | """
514 | lines = []
515 |
516 | # Header
517 | status = "✅ PASSED" if validation_results.get("passed", False) else "❌ FAILED"
518 | lines.append(f"=== Data Validation Report - {status} ===")
519 | lines.append("")
520 |
521 | # Metrics
522 | if "metrics" in validation_results:
523 | lines.append("📊 Metrics:")
524 | for key, value in validation_results["metrics"].items():
525 | if isinstance(value, float) and 0 < value < 1:
526 | lines.append(f" • {key}: {value:.2%}")
527 | else:
528 | lines.append(f" • {key}: {value}")
529 | lines.append("")
530 |
531 | # Errors
532 | if validation_results.get("errors"):
533 | lines.append("❌ Errors:")
534 | for error in validation_results["errors"]:
535 | lines.append(f" • {error}")
536 | lines.append("")
537 |
538 | # Warnings
539 | if include_warnings and validation_results.get("warnings"):
540 | lines.append("⚠️ Warnings:")
541 | for warning in validation_results["warnings"]:
542 | lines.append(f" • {warning}")
543 | lines.append("")
544 |
545 | # Symbol-specific results (for batch validation)
546 | if "symbol_results" in validation_results:
547 | failed_symbols = [
548 | symbol
549 | for symbol, result in validation_results["symbol_results"].items()
550 | if not result.get("passed", True)
551 | ]
552 | if failed_symbols:
553 | lines.append(f"🔍 Failed Symbols ({len(failed_symbols)}):")
554 | for symbol in failed_symbols:
555 | result = validation_results["symbol_results"][symbol]
556 | error = result.get("error", "Unknown error")
557 | lines.append(f" • {symbol}: {error}")
558 | lines.append("")
559 |
560 | return "\n".join(lines)
561 |
562 |
563 | # Convenience functions for common validation scenarios
564 | def validate_stock_data(
565 | data: DataFrame,
566 | symbol: str,
567 | start_date: str | None = None,
568 | end_date: str | None = None,
569 | strict: bool = True,
570 | ) -> dict[str, Any]:
571 | """
572 | Convenience function to validate stock data with date range.
573 |
574 | Args:
575 | data: Stock price DataFrame
576 | symbol: Stock symbol
577 | start_date: Expected start date (optional)
578 | end_date: Expected end date (optional)
579 | strict: Whether to use strict validation
580 |
581 | Returns:
582 | Combined validation results
583 | """
584 | validator = DataValidator()
585 |
586 | # Validate price data
587 | price_results = validator.validate_price_data(data, symbol, strict_mode=strict)
588 |
589 | # Validate date range if provided
590 | if start_date and end_date:
591 | try:
592 | validator.validate_date_range(start_date, end_date)
593 | price_results["date_range_valid"] = True
594 | except ValidationError as e:
595 | price_results["date_range_valid"] = False
596 | price_results["errors"].append(f"Date range validation failed: {e}")
597 | price_results["passed"] = False
598 |
599 | return price_results
600 |
601 |
602 | def validate_backtest_data(
603 | data: dict[str, DataFrame], min_history_days: int = 30
604 | ) -> dict[str, Any]:
605 | """
606 | Convenience function to validate backtesting data requirements.
607 |
608 | Args:
609 | data: Dictionary of symbol -> DataFrame mappings
610 | min_history_days: Minimum days of history required
611 |
612 | Returns:
613 | Validation results for backtesting
614 | """
615 | validator = DataValidator()
616 |
617 | # Validate batch data
618 | batch_results = validator.validate_batch_data(data, validate_individual=True)
619 |
620 | # Additional backtesting-specific checks
621 | for symbol, df in data.items():
622 | if not df.empty and len(df) < min_history_days:
623 | warning_msg = (
624 | f"{symbol}: Only {len(df)} days of data (minimum: {min_history_days})"
625 | )
626 | batch_results["warnings"].append(warning_msg)
627 |
628 | return batch_results
629 |
```
--------------------------------------------------------------------------------
/maverick_mcp/workflows/agents/validator_agent.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Validator Agent for backtesting results validation and robustness testing.
3 |
4 | This agent performs walk-forward analysis, Monte Carlo simulation, and robustness
5 | testing to validate optimization results and provide confidence-scored recommendations.
6 | """
7 |
8 | import logging
9 | import statistics
10 | from datetime import datetime, timedelta
11 | from typing import Any
12 |
13 | from maverick_mcp.backtesting import StrategyOptimizer, VectorBTEngine
14 | from maverick_mcp.workflows.state import BacktestingWorkflowState
15 |
16 | logger = logging.getLogger(__name__)
17 |
18 |
19 | class ValidatorAgent:
20 | """Intelligent validator for backtesting results and strategy robustness."""
21 |
22 | def __init__(
23 | self,
24 | vectorbt_engine: VectorBTEngine | None = None,
25 | strategy_optimizer: StrategyOptimizer | None = None,
26 | ):
27 | """Initialize validator agent.
28 |
29 | Args:
30 | vectorbt_engine: VectorBT backtesting engine
31 | strategy_optimizer: Strategy optimization engine
32 | """
33 | self.engine = vectorbt_engine or VectorBTEngine()
34 | self.optimizer = strategy_optimizer or StrategyOptimizer(self.engine)
35 |
36 | # Validation criteria for different regimes
37 | self.REGIME_VALIDATION_CRITERIA = {
38 | "trending": {
39 | "min_sharpe_ratio": 0.8,
40 | "max_drawdown_threshold": 0.25,
41 | "min_total_return": 0.10,
42 | "min_win_rate": 0.35,
43 | "stability_threshold": 0.7,
44 | },
45 | "ranging": {
46 | "min_sharpe_ratio": 1.0, # Higher standard for ranging markets
47 | "max_drawdown_threshold": 0.15,
48 | "min_total_return": 0.05,
49 | "min_win_rate": 0.45,
50 | "stability_threshold": 0.8,
51 | },
52 | "volatile": {
53 | "min_sharpe_ratio": 0.6, # Lower expectation in volatile markets
54 | "max_drawdown_threshold": 0.35,
55 | "min_total_return": 0.08,
56 | "min_win_rate": 0.30,
57 | "stability_threshold": 0.6,
58 | },
59 | "volatile_trending": {
60 | "min_sharpe_ratio": 0.7,
61 | "max_drawdown_threshold": 0.30,
62 | "min_total_return": 0.12,
63 | "min_win_rate": 0.35,
64 | "stability_threshold": 0.65,
65 | },
66 | "low_volume": {
67 | "min_sharpe_ratio": 0.9,
68 | "max_drawdown_threshold": 0.20,
69 | "min_total_return": 0.06,
70 | "min_win_rate": 0.40,
71 | "stability_threshold": 0.75,
72 | },
73 | "unknown": {
74 | "min_sharpe_ratio": 0.8,
75 | "max_drawdown_threshold": 0.20,
76 | "min_total_return": 0.08,
77 | "min_win_rate": 0.40,
78 | "stability_threshold": 0.7,
79 | },
80 | }
81 |
82 | # Robustness scoring weights
83 | self.ROBUSTNESS_WEIGHTS = {
84 | "walk_forward_consistency": 0.3,
85 | "parameter_sensitivity": 0.2,
86 | "monte_carlo_stability": 0.2,
87 | "out_of_sample_performance": 0.3,
88 | }
89 |
90 | logger.info("ValidatorAgent initialized")
91 |
92 | async def validate_strategies(
93 | self, state: BacktestingWorkflowState
94 | ) -> BacktestingWorkflowState:
95 | """Validate optimized strategies through comprehensive testing.
96 |
97 | Args:
98 | state: Current workflow state with optimization results
99 |
100 | Returns:
101 | Updated state with validation results and final recommendations
102 | """
103 | start_time = datetime.now()
104 |
105 | try:
106 | logger.info(
107 | f"Validating {len(state.best_parameters)} strategies for {state.symbol}"
108 | )
109 |
110 | # Get validation criteria for current regime
111 | validation_criteria = self._get_validation_criteria(state.market_regime)
112 |
113 | # Perform validation for each strategy
114 | walk_forward_results = {}
115 | monte_carlo_results = {}
116 | out_of_sample_performance = {}
117 | robustness_scores = {}
118 | validation_warnings = []
119 |
120 | for strategy, parameters in state.best_parameters.items():
121 | try:
122 | logger.info(f"Validating {strategy} strategy...")
123 |
124 | # Walk-forward analysis
125 | wf_result = await self._run_walk_forward_analysis(
126 | state, strategy, parameters
127 | )
128 | walk_forward_results[strategy] = wf_result
129 |
130 | # Monte Carlo simulation
131 | mc_result = await self._run_monte_carlo_simulation(
132 | state, strategy, parameters
133 | )
134 | monte_carlo_results[strategy] = mc_result
135 |
136 | # Out-of-sample testing
137 | oos_result = await self._run_out_of_sample_test(
138 | state, strategy, parameters
139 | )
140 | out_of_sample_performance[strategy] = oos_result
141 |
142 | # Calculate robustness score
143 | robustness_score = self._calculate_robustness_score(
144 | wf_result, mc_result, oos_result, validation_criteria
145 | )
146 | robustness_scores[strategy] = robustness_score
147 |
148 | # Check for validation warnings
149 | warnings = self._check_validation_warnings(
150 | strategy, wf_result, mc_result, oos_result, validation_criteria
151 | )
152 | validation_warnings.extend(warnings)
153 |
154 | logger.info(
155 | f"Validated {strategy}: robustness score {robustness_score:.2f}"
156 | )
157 |
158 | except Exception as e:
159 | logger.error(f"Failed to validate {strategy}: {e}")
160 | robustness_scores[strategy] = 0.0
161 | validation_warnings.append(
162 | f"{strategy}: Validation failed - {str(e)}"
163 | )
164 |
165 | # Generate final recommendations
166 | final_ranking = self._generate_final_ranking(
167 | state.best_parameters, robustness_scores, state.strategy_rankings
168 | )
169 |
170 | # Select recommended strategy
171 | recommended_strategy, recommendation_confidence = (
172 | self._select_recommended_strategy(
173 | final_ranking, robustness_scores, state.regime_confidence
174 | )
175 | )
176 |
177 | # Perform risk assessment
178 | risk_assessment = self._perform_risk_assessment(
179 | recommended_strategy,
180 | walk_forward_results,
181 | monte_carlo_results,
182 | validation_criteria,
183 | )
184 |
185 | # Update state
186 | state.walk_forward_results = walk_forward_results
187 | state.monte_carlo_results = monte_carlo_results
188 | state.out_of_sample_performance = out_of_sample_performance
189 | state.robustness_score = robustness_scores
190 | state.validation_warnings = validation_warnings
191 | state.final_strategy_ranking = final_ranking
192 | state.recommended_strategy = recommended_strategy
193 | state.recommended_parameters = state.best_parameters.get(
194 | recommended_strategy, {}
195 | )
196 | state.recommendation_confidence = recommendation_confidence
197 | state.risk_assessment = risk_assessment
198 |
199 | # Update workflow status
200 | state.workflow_status = "completed"
201 | state.current_step = "validation_completed"
202 | state.steps_completed.append("strategy_validation")
203 |
204 | # Record total execution time
205 | total_execution_time = (datetime.now() - start_time).total_seconds() * 1000
206 | state.total_execution_time_ms = (
207 | state.regime_analysis_time_ms
208 | + state.optimization_time_ms
209 | + total_execution_time
210 | )
211 |
212 | logger.info(
213 | f"Strategy validation completed for {state.symbol}: "
214 | f"Recommended {recommended_strategy} with confidence {recommendation_confidence:.2f}"
215 | )
216 |
217 | return state
218 |
219 | except Exception as e:
220 | error_info = {
221 | "step": "strategy_validation",
222 | "error": str(e),
223 | "timestamp": datetime.now().isoformat(),
224 | "symbol": state.symbol,
225 | }
226 | state.errors_encountered.append(error_info)
227 |
228 | # Fallback recommendation
229 | if state.best_parameters:
230 | fallback_strategy = list(state.best_parameters.keys())[0]
231 | state.recommended_strategy = fallback_strategy
232 | state.recommended_parameters = state.best_parameters[fallback_strategy]
233 | state.recommendation_confidence = 0.3
234 | state.fallback_strategies_used.append("validation_fallback")
235 |
236 | logger.error(f"Strategy validation failed for {state.symbol}: {e}")
237 | return state
238 |
239 | def _get_validation_criteria(self, regime: str) -> dict[str, Any]:
240 | """Get validation criteria based on market regime."""
241 | return self.REGIME_VALIDATION_CRITERIA.get(
242 | regime, self.REGIME_VALIDATION_CRITERIA["unknown"]
243 | )
244 |
245 | async def _run_walk_forward_analysis(
246 | self, state: BacktestingWorkflowState, strategy: str, parameters: dict[str, Any]
247 | ) -> dict[str, Any]:
248 | """Run walk-forward analysis for strategy validation."""
249 | try:
250 | # Calculate walk-forward windows
251 | start_dt = datetime.strptime(state.start_date, "%Y-%m-%d")
252 | end_dt = datetime.strptime(state.end_date, "%Y-%m-%d")
253 | total_days = (end_dt - start_dt).days
254 |
255 | # Use appropriate window sizes based on data length
256 | if total_days > 500: # ~2 years
257 | window_size = 252 # 1 year
258 | step_size = 63 # 3 months
259 | elif total_days > 250: # ~1 year
260 | window_size = 126 # 6 months
261 | step_size = 42 # 6 weeks
262 | else:
263 | window_size = 63 # 3 months
264 | step_size = 21 # 3 weeks
265 |
266 | # Run walk-forward analysis using the optimizer
267 | wf_result = await self.optimizer.walk_forward_analysis(
268 | symbol=state.symbol,
269 | strategy_type=strategy,
270 | parameters=parameters,
271 | start_date=state.start_date,
272 | end_date=state.end_date,
273 | window_size=window_size,
274 | step_size=step_size,
275 | )
276 |
277 | return wf_result
278 |
279 | except Exception as e:
280 | logger.error(f"Walk-forward analysis failed for {strategy}: {e}")
281 | return {"error": str(e), "consistency_score": 0.0}
282 |
283 | async def _run_monte_carlo_simulation(
284 | self, state: BacktestingWorkflowState, strategy: str, parameters: dict[str, Any]
285 | ) -> dict[str, Any]:
286 | """Run Monte Carlo simulation for strategy validation."""
287 | try:
288 | # First run a backtest to get base results
289 | backtest_result = await self.engine.run_backtest(
290 | symbol=state.symbol,
291 | strategy_type=strategy,
292 | parameters=parameters,
293 | start_date=state.start_date,
294 | end_date=state.end_date,
295 | initial_capital=state.initial_capital,
296 | )
297 |
298 | # Run Monte Carlo simulation
299 | mc_result = await self.optimizer.monte_carlo_simulation(
300 | backtest_results=backtest_result,
301 | num_simulations=500, # Reduced for performance
302 | )
303 |
304 | return mc_result
305 |
306 | except Exception as e:
307 | logger.error(f"Monte Carlo simulation failed for {strategy}: {e}")
308 | return {"error": str(e), "stability_score": 0.0}
309 |
310 | async def _run_out_of_sample_test(
311 | self, state: BacktestingWorkflowState, strategy: str, parameters: dict[str, Any]
312 | ) -> dict[str, float]:
313 | """Run out-of-sample testing on holdout data."""
314 | try:
315 | # Use last 30% of data as out-of-sample
316 | start_dt = datetime.strptime(state.start_date, "%Y-%m-%d")
317 | end_dt = datetime.strptime(state.end_date, "%Y-%m-%d")
318 | total_days = (end_dt - start_dt).days
319 |
320 | oos_days = int(total_days * 0.3)
321 | oos_start = end_dt - timedelta(days=oos_days)
322 |
323 | # Run backtest on out-of-sample period
324 | oos_result = await self.engine.run_backtest(
325 | symbol=state.symbol,
326 | strategy_type=strategy,
327 | parameters=parameters,
328 | start_date=oos_start.strftime("%Y-%m-%d"),
329 | end_date=state.end_date,
330 | initial_capital=state.initial_capital,
331 | )
332 |
333 | return {
334 | "total_return": oos_result["metrics"]["total_return"],
335 | "sharpe_ratio": oos_result["metrics"]["sharpe_ratio"],
336 | "max_drawdown": oos_result["metrics"]["max_drawdown"],
337 | "win_rate": oos_result["metrics"]["win_rate"],
338 | "total_trades": oos_result["metrics"]["total_trades"],
339 | }
340 |
341 | except Exception as e:
342 | logger.error(f"Out-of-sample test failed for {strategy}: {e}")
343 | return {
344 | "total_return": 0.0,
345 | "sharpe_ratio": 0.0,
346 | "max_drawdown": 0.0,
347 | "win_rate": 0.0,
348 | "total_trades": 0,
349 | }
350 |
351 | def _calculate_robustness_score(
352 | self,
353 | wf_result: dict[str, Any],
354 | mc_result: dict[str, Any],
355 | oos_result: dict[str, float],
356 | validation_criteria: dict[str, Any],
357 | ) -> float:
358 | """Calculate overall robustness score for a strategy."""
359 | scores = {}
360 |
361 | # Walk-forward consistency score
362 | if "consistency_score" in wf_result:
363 | scores["walk_forward_consistency"] = wf_result["consistency_score"]
364 | elif "error" not in wf_result and "periods" in wf_result:
365 | # Calculate consistency from period results
366 | period_returns = [
367 | p.get("total_return", 0) for p in wf_result.get("periods", [])
368 | ]
369 | if period_returns:
370 | # Lower std deviation relative to mean = higher consistency
371 | mean_return = statistics.mean(period_returns)
372 | std_return = (
373 | statistics.stdev(period_returns) if len(period_returns) > 1 else 0
374 | )
375 | consistency = max(0, 1 - (std_return / max(abs(mean_return), 0.01)))
376 | scores["walk_forward_consistency"] = min(1.0, consistency)
377 | else:
378 | scores["walk_forward_consistency"] = 0.0
379 | else:
380 | scores["walk_forward_consistency"] = 0.0
381 |
382 | # Parameter sensitivity (inverse of standard error)
383 | scores["parameter_sensitivity"] = 0.7 # Default moderate sensitivity
384 |
385 | # Monte Carlo stability
386 | if "stability_score" in mc_result:
387 | scores["monte_carlo_stability"] = mc_result["stability_score"]
388 | elif "error" not in mc_result and "percentiles" in mc_result:
389 | # Calculate stability from percentile spread
390 | percentiles = mc_result["percentiles"]
391 | p10 = percentiles.get("10", 0)
392 | p90 = percentiles.get("90", 0)
393 | median = percentiles.get("50", 0)
394 |
395 | if median != 0:
396 | stability = 1 - abs(p90 - p10) / abs(median)
397 | scores["monte_carlo_stability"] = max(0, min(1, stability))
398 | else:
399 | scores["monte_carlo_stability"] = 0.0
400 | else:
401 | scores["monte_carlo_stability"] = 0.0
402 |
403 | # Out-of-sample performance score
404 | oos_score = 0.0
405 | if oos_result["sharpe_ratio"] >= validation_criteria["min_sharpe_ratio"]:
406 | oos_score += 0.3
407 | if (
408 | abs(oos_result["max_drawdown"])
409 | <= validation_criteria["max_drawdown_threshold"]
410 | ):
411 | oos_score += 0.3
412 | if oos_result["total_return"] >= validation_criteria["min_total_return"]:
413 | oos_score += 0.2
414 | if oos_result["win_rate"] >= validation_criteria["min_win_rate"]:
415 | oos_score += 0.2
416 |
417 | scores["out_of_sample_performance"] = oos_score
418 |
419 | # Calculate weighted robustness score
420 | robustness_score = sum(
421 | scores[component] * self.ROBUSTNESS_WEIGHTS[component]
422 | for component in self.ROBUSTNESS_WEIGHTS
423 | )
424 |
425 | return max(0.0, min(1.0, robustness_score))
426 |
427 | def _check_validation_warnings(
428 | self,
429 | strategy: str,
430 | wf_result: dict[str, Any],
431 | mc_result: dict[str, Any],
432 | oos_result: dict[str, float],
433 | validation_criteria: dict[str, Any],
434 | ) -> list[str]:
435 | """Check for validation warnings and concerns."""
436 | warnings = []
437 |
438 | # Walk-forward analysis warnings
439 | if "error" in wf_result:
440 | warnings.append(f"{strategy}: Walk-forward analysis failed")
441 | elif (
442 | wf_result.get("consistency_score", 0)
443 | < validation_criteria["stability_threshold"]
444 | ):
445 | warnings.append(
446 | f"{strategy}: Low walk-forward consistency ({wf_result.get('consistency_score', 0):.2f})"
447 | )
448 |
449 | # Monte Carlo warnings
450 | if "error" in mc_result:
451 | warnings.append(f"{strategy}: Monte Carlo simulation failed")
452 | elif mc_result.get("stability_score", 0) < 0.6:
453 | warnings.append(f"{strategy}: High Monte Carlo variability")
454 |
455 | # Out-of-sample warnings
456 | if oos_result["total_trades"] < 5:
457 | warnings.append(
458 | f"{strategy}: Very few out-of-sample trades ({oos_result['total_trades']})"
459 | )
460 |
461 | if oos_result["sharpe_ratio"] < validation_criteria["min_sharpe_ratio"]:
462 | warnings.append(
463 | f"{strategy}: Low out-of-sample Sharpe ratio ({oos_result['sharpe_ratio']:.2f})"
464 | )
465 |
466 | if (
467 | abs(oos_result["max_drawdown"])
468 | > validation_criteria["max_drawdown_threshold"]
469 | ):
470 | warnings.append(
471 | f"{strategy}: High out-of-sample drawdown ({oos_result['max_drawdown']:.2f})"
472 | )
473 |
474 | return warnings
475 |
476 | def _generate_final_ranking(
477 | self,
478 | best_parameters: dict[str, dict[str, Any]],
479 | robustness_scores: dict[str, float],
480 | strategy_rankings: dict[str, float],
481 | ) -> list[dict[str, Any]]:
482 | """Generate final ranked recommendations."""
483 | rankings = []
484 |
485 | for strategy in best_parameters.keys():
486 | robustness = robustness_scores.get(strategy, 0.0)
487 | fitness = strategy_rankings.get(strategy, 0.5)
488 |
489 | # Combined score: 60% robustness, 40% initial fitness
490 | combined_score = robustness * 0.6 + fitness * 0.4
491 |
492 | rankings.append(
493 | {
494 | "strategy": strategy,
495 | "robustness_score": robustness,
496 | "fitness_score": fitness,
497 | "combined_score": combined_score,
498 | "parameters": best_parameters[strategy],
499 | "recommendation": self._get_recommendation_level(combined_score),
500 | }
501 | )
502 |
503 | # Sort by combined score
504 | rankings.sort(key=lambda x: x["combined_score"], reverse=True)
505 |
506 | return rankings
507 |
508 | def _get_recommendation_level(self, combined_score: float) -> str:
509 | """Get recommendation level based on combined score."""
510 | if combined_score >= 0.8:
511 | return "Highly Recommended"
512 | elif combined_score >= 0.6:
513 | return "Recommended"
514 | elif combined_score >= 0.4:
515 | return "Acceptable"
516 | else:
517 | return "Not Recommended"
518 |
519 | def _select_recommended_strategy(
520 | self,
521 | final_ranking: list[dict[str, Any]],
522 | robustness_scores: dict[str, float],
523 | regime_confidence: float,
524 | ) -> tuple[str, float]:
525 | """Select the final recommended strategy and calculate confidence."""
526 | if not final_ranking:
527 | return "sma_cross", 0.1 # Fallback
528 |
529 | # Select top strategy
530 | top_strategy = final_ranking[0]["strategy"]
531 | top_score = final_ranking[0]["combined_score"]
532 |
533 | # Calculate recommendation confidence
534 | confidence_factors = []
535 |
536 | # Score-based confidence
537 | confidence_factors.append(top_score)
538 |
539 | # Robustness-based confidence
540 | robustness = robustness_scores.get(top_strategy, 0.0)
541 | confidence_factors.append(robustness)
542 |
543 | # Regime confidence factor
544 | confidence_factors.append(regime_confidence)
545 |
546 | # Score separation from second-best
547 | if len(final_ranking) > 1:
548 | score_gap = top_score - final_ranking[1]["combined_score"]
549 | separation_confidence = min(score_gap * 2, 1.0) # Scale to 0-1
550 | confidence_factors.append(separation_confidence)
551 | else:
552 | confidence_factors.append(0.5) # Moderate confidence for single option
553 |
554 | # Calculate overall confidence
555 | recommendation_confidence = sum(confidence_factors) / len(confidence_factors)
556 | recommendation_confidence = max(0.1, min(0.95, recommendation_confidence))
557 |
558 | return top_strategy, recommendation_confidence
559 |
560 | def _perform_risk_assessment(
561 | self,
562 | recommended_strategy: str,
563 | walk_forward_results: dict[str, dict[str, Any]],
564 | monte_carlo_results: dict[str, dict[str, Any]],
565 | validation_criteria: dict[str, Any],
566 | ) -> dict[str, Any]:
567 | """Perform comprehensive risk assessment of recommended strategy."""
568 | wf_result = walk_forward_results.get(recommended_strategy, {})
569 | mc_result = monte_carlo_results.get(recommended_strategy, {})
570 |
571 | risk_assessment = {
572 | "overall_risk_level": "Medium",
573 | "key_risks": [],
574 | "risk_mitigation": [],
575 | "confidence_intervals": {},
576 | "worst_case_scenario": {},
577 | }
578 |
579 | # Analyze walk-forward results for risk patterns
580 | if "periods" in wf_result:
581 | periods = wf_result["periods"]
582 | negative_periods = [p for p in periods if p.get("total_return", 0) < 0]
583 |
584 | if len(negative_periods) / len(periods) > 0.4:
585 | risk_assessment["key_risks"].append("High frequency of losing periods")
586 | risk_assessment["overall_risk_level"] = "High"
587 |
588 | max_period_loss = min([p.get("total_return", 0) for p in periods])
589 | if max_period_loss < -0.15:
590 | risk_assessment["key_risks"].append(
591 | f"Severe single-period loss: {max_period_loss:.1%}"
592 | )
593 |
594 | # Analyze Monte Carlo results
595 | if "percentiles" in mc_result:
596 | percentiles = mc_result["percentiles"]
597 | worst_case = percentiles.get("5", 0) # 5th percentile
598 |
599 | risk_assessment["worst_case_scenario"] = {
600 | "return_5th_percentile": worst_case,
601 | "probability": 0.05,
602 | "description": f"5% chance of returns below {worst_case:.1%}",
603 | }
604 |
605 | risk_assessment["confidence_intervals"] = {
606 | "90_percent_range": f"{percentiles.get('5', 0):.1%} to {percentiles.get('95', 0):.1%}",
607 | "median_return": f"{percentiles.get('50', 0):.1%}",
608 | }
609 |
610 | # Risk mitigation recommendations
611 | risk_assessment["risk_mitigation"] = [
612 | "Use position sizing based on volatility",
613 | "Implement stop-loss orders",
614 | "Monitor strategy performance regularly",
615 | "Consider diversification across multiple strategies",
616 | ]
617 |
618 | return risk_assessment
619 |
```
--------------------------------------------------------------------------------
/tests/test_production_validation.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Production Validation Test Suite for MaverickMCP.
3 |
4 | This suite validates that the system is ready for production deployment
5 | by testing configuration, environment setup, monitoring, backup procedures,
6 | and production-like load scenarios.
7 |
8 | Validates:
9 | - Environment configuration correctness
10 | - SSL/TLS configuration (when available)
11 | - Monitoring and alerting systems
12 | - Backup and recovery procedures
13 | - Load testing with production-like scenarios
14 | - Security configuration in production mode
15 | - Database migration status and integrity
16 | - Performance optimization effectiveness
17 | """
18 |
19 | import asyncio
20 | import os
21 | import ssl
22 | import time
23 | from pathlib import Path
24 | from unittest.mock import patch
25 |
26 | import pytest
27 | from fastapi.testclient import TestClient
28 |
29 | from maverick_mcp.api.api_server import create_api_app
30 | from maverick_mcp.config.settings import get_settings
31 | from maverick_mcp.config.validation import get_validation_status
32 | from maverick_mcp.data.models import SessionLocal
33 | from maverick_mcp.data.performance import (
34 | cleanup_performance_systems,
35 | get_performance_metrics,
36 | initialize_performance_systems,
37 | )
38 | from maverick_mcp.utils.monitoring import get_metrics, initialize_monitoring
39 |
40 |
41 | @pytest.fixture(scope="session")
42 | def production_settings():
43 | """Get production-like settings."""
44 | with patch.dict(
45 | os.environ,
46 | {
47 | "ENVIRONMENT": "production",
48 | "AUTH_ENABLED": "true",
49 | "SECURITY_ENABLED": "true",
50 | "JWT_SECRET": "test-jwt-secret-for-production-validation-tests-minimum-32-chars",
51 | "DATABASE_URL": "postgresql://test:test@localhost/test_prod_db",
52 | },
53 | ):
54 | return get_settings()
55 |
56 |
57 | @pytest.fixture
58 | def production_app(production_settings):
59 | """Create production-configured app."""
60 | return create_api_app()
61 |
62 |
63 | @pytest.fixture
64 | def production_client(production_app):
65 | """Create client for production testing."""
66 | return TestClient(production_app)
67 |
68 |
69 | class TestEnvironmentConfiguration:
70 | """Test production environment configuration."""
71 |
72 | @pytest.mark.skip(reason="Incompatible with global test environment configuration")
73 | def test_environment_variables_set(self, production_settings):
74 | """Test that all required environment variables are set."""
75 |
76 | # Critical environment variables for production
77 | critical_vars = [
78 | "DATABASE_URL",
79 | "JWT_SECRET",
80 | "ENVIRONMENT",
81 | ]
82 |
83 | # Check that critical vars are set (not default values)
84 | for var in critical_vars:
85 | env_value = os.getenv(var)
86 | if var == "DATABASE_URL":
87 | # Should not be default SQLite in production
88 | if env_value is None:
89 | pytest.skip(f"{var} not set in test environment")
90 | if env_value:
91 | assert (
92 | "sqlite" not in env_value.lower()
93 | or "memory" not in env_value.lower()
94 | )
95 |
96 | elif var == "JWT_SECRET":
97 | # Should not be default/weak secret
98 | if env_value is None:
99 | pytest.skip(f"{var} not set in test environment")
100 | if env_value:
101 | assert len(env_value) >= 32
102 | assert env_value != "your-secret-key-here"
103 | assert env_value != "development-key"
104 |
105 | elif var == "ENVIRONMENT":
106 | if env_value is None:
107 | pytest.skip(f"{var} not set in test environment")
108 | assert env_value in ["production", "staging"]
109 |
110 | def test_security_configuration(self, production_settings):
111 | """Test security configuration for production."""
112 |
113 | # Authentication should be enabled
114 | assert production_settings.auth.enabled is True
115 |
116 | # Secure cookies in production
117 | if production_settings.environment == "production":
118 | # Cookie security should be enabled (skip if not implemented)
119 | if not hasattr(production_settings, "cookie_secure"):
120 | pytest.skip("Cookie secure setting not implemented yet")
121 |
122 | # JWT configuration
123 | assert production_settings.auth.jwt_algorithm in ["RS256", "HS256"]
124 | assert (
125 | production_settings.auth.jwt_access_token_expire_minutes <= 60
126 | ) # Not too long
127 |
128 | # Redis configuration (should not use default)
129 | if hasattr(production_settings.auth, "redis_url"):
130 | redis_url = production_settings.auth.redis_url
131 | assert "localhost" not in redis_url or os.getenv("REDIS_HOST") is not None
132 |
133 | def test_database_configuration(self, production_settings):
134 | """Test database configuration for production."""
135 |
136 | # Get database URL from environment or settings
137 | database_url = os.getenv("DATABASE_URL", "")
138 | if not database_url:
139 | pytest.skip("DATABASE_URL not set in environment")
140 |
141 | # Should use production database (not SQLite)
142 | assert (
143 | "postgresql" in database_url.lower() or "mysql" in database_url.lower()
144 | ) and "sqlite" not in database_url.lower()
145 |
146 | # Should not use default credentials
147 | if "postgresql://" in database_url:
148 | assert "password" not in database_url or "your-password" not in database_url
149 | assert (
150 | "localhost" not in database_url
151 | or os.getenv("DATABASE_HOST") is not None
152 | )
153 |
154 | # Test database connection
155 | try:
156 | with SessionLocal() as session:
157 | result = session.execute("SELECT 1")
158 | assert result.scalar() == 1
159 | except Exception as e:
160 | pytest.skip(f"Database connection test skipped: {e}")
161 |
162 | def test_logging_configuration(self, production_settings):
163 | """Test logging configuration for production."""
164 |
165 | # Log level should be appropriate for production
166 | assert production_settings.api.log_level.upper() in ["INFO", "WARNING", "ERROR"]
167 |
168 | # Should not be DEBUG in production
169 | if production_settings.environment == "production":
170 | assert production_settings.api.log_level.upper() != "DEBUG"
171 |
172 | def test_api_configuration(self, production_settings):
173 | """Test API configuration for production."""
174 |
175 | # Debug features should be disabled
176 | if production_settings.environment == "production":
177 | assert production_settings.api.debug is False
178 |
179 | # CORS should be properly configured
180 | cors_origins = production_settings.api.cors_origins
181 | assert cors_origins is not None
182 |
183 | # Should not allow all origins in production
184 | if production_settings.environment == "production":
185 | assert "*" not in cors_origins
186 |
187 |
188 | class TestSystemValidation:
189 | """Test system validation and health checks."""
190 |
191 | def test_configuration_validation(self):
192 | """Test configuration validation system."""
193 |
194 | validation_status = get_validation_status()
195 |
196 | # Should have validation status
197 | assert "valid" in validation_status
198 | assert "warnings" in validation_status
199 | assert "errors" in validation_status
200 |
201 | # In production, should have minimal warnings/errors
202 | if os.getenv("ENVIRONMENT") == "production":
203 | assert len(validation_status["errors"]) == 0
204 | assert len(validation_status["warnings"]) <= 2 # Allow some minor warnings
205 |
206 | def test_health_check_endpoint(self, production_client):
207 | """Test health check endpoint functionality."""
208 |
209 | response = production_client.get("/health")
210 | assert response.status_code == 200
211 |
212 | health_data = response.json()
213 | assert "status" in health_data
214 | assert health_data["status"] in ["healthy", "degraded"]
215 |
216 | # Should include service information
217 | assert "services" in health_data
218 | assert "version" in health_data
219 |
220 | # Should include circuit breakers
221 | assert "circuit_breakers" in health_data
222 |
223 | @pytest.mark.integration
224 | def test_database_health(self):
225 | """Test database health and connectivity."""
226 |
227 | try:
228 | with SessionLocal() as session:
229 | # Test basic connectivity
230 | from sqlalchemy import text
231 |
232 | result = session.execute(text("SELECT 1 as health_check"))
233 | assert result.scalar() == 1
234 |
235 | # Test transaction capability
236 | # Session already has a transaction, so just test query
237 | # Use SQLite-compatible query for testing
238 | result = session.execute(
239 | text("SELECT COUNT(*) FROM sqlite_master WHERE type='table'")
240 | if "sqlite" in str(session.bind.url)
241 | else text("SELECT COUNT(*) FROM information_schema.tables")
242 | )
243 | assert result.scalar() >= 0 # Should return some count
244 |
245 | except Exception as e:
246 | pytest.fail(f"Database health check failed: {e}")
247 |
248 | @pytest.mark.asyncio
249 | @pytest.mark.integration
250 | async def test_performance_systems_health(self):
251 | """Test performance systems health."""
252 |
253 | # Initialize performance systems
254 | performance_status = await initialize_performance_systems()
255 |
256 | # Should initialize successfully
257 | assert isinstance(performance_status, dict)
258 | assert "redis_manager" in performance_status
259 |
260 | # Get performance metrics
261 | metrics = await get_performance_metrics()
262 | assert "redis_manager" in metrics
263 | assert "request_cache" in metrics
264 | assert "query_optimizer" in metrics
265 | assert "timestamp" in metrics
266 |
267 | # Cleanup
268 | await cleanup_performance_systems()
269 |
270 | def test_monitoring_systems(self):
271 | """Test monitoring systems are functional."""
272 |
273 | try:
274 | # Initialize monitoring
275 | initialize_monitoring()
276 |
277 | # Get metrics
278 | metrics_data = get_metrics()
279 | assert isinstance(metrics_data, str)
280 |
281 | # Should be Prometheus format
282 | assert (
283 | "# HELP" in metrics_data
284 | or "# TYPE" in metrics_data
285 | or len(metrics_data) > 0
286 | )
287 |
288 | except Exception as e:
289 | pytest.skip(f"Monitoring test skipped: {e}")
290 |
291 |
292 | class TestSSLTLSConfiguration:
293 | """Test SSL/TLS configuration (when available)."""
294 |
295 | def test_ssl_certificate_validity(self):
296 | """Test SSL certificate validity."""
297 |
298 | # This would test actual SSL certificate in production
299 | # For testing, we check if SSL context can be created
300 |
301 | try:
302 | context = ssl.create_default_context()
303 | assert context.check_hostname is True
304 | assert context.verify_mode == ssl.CERT_REQUIRED
305 |
306 | except Exception as e:
307 | pytest.skip(f"SSL test skipped: {e}")
308 |
309 | def test_tls_configuration(self, production_client):
310 | """Test TLS configuration."""
311 |
312 | # Test security headers are present
313 | production_client.get("/health")
314 |
315 | # Should have security headers in production
316 | security_headers = [
317 | "X-Content-Type-Options",
318 | "X-Frame-Options",
319 | "X-XSS-Protection",
320 | ]
321 |
322 | # Note: These would be set by security middleware
323 | # Check if security middleware is active
324 | for _header in security_headers:
325 | # In test environment, headers might not be set
326 | # In production, they should be present
327 | if os.getenv("ENVIRONMENT") == "production":
328 | # assert header in response.headers
329 | pass # Skip for test environment
330 |
331 | def test_secure_cookie_configuration(self, production_client, production_settings):
332 | """Test secure cookie configuration."""
333 |
334 | if production_settings.environment != "production":
335 | pytest.skip("Secure cookie test only for production")
336 |
337 | # Test that cookies are set with secure flags
338 | test_user = {
339 | "email": "[email protected]",
340 | "password": "TestPass123!",
341 | "name": "SSL Test User",
342 | }
343 |
344 | # Register and login
345 | production_client.post("/auth/register", json=test_user)
346 | login_response = production_client.post(
347 | "/auth/login",
348 | json={"email": test_user["email"], "password": test_user["password"]},
349 | )
350 |
351 | # Check cookie headers for security flags
352 | cookie_header = login_response.headers.get("set-cookie", "")
353 | if cookie_header:
354 | # Should have Secure flag in production
355 | assert "Secure" in cookie_header
356 | assert "HttpOnly" in cookie_header
357 | assert "SameSite" in cookie_header
358 |
359 |
360 | class TestBackupAndRecovery:
361 | """Test backup and recovery procedures."""
362 |
363 | def test_database_backup_capability(self):
364 | """Test database backup capability."""
365 |
366 | try:
367 | with SessionLocal() as session:
368 | # Test that we can read critical tables
369 | critical_tables = [
370 | "mcp_users",
371 | "mcp_api_keys",
372 | "auth_audit_log",
373 | ]
374 |
375 | for table in critical_tables:
376 | try:
377 | result = session.execute(f"SELECT COUNT(*) FROM {table}")
378 | count = result.scalar()
379 | assert count >= 0 # Should be able to count rows
380 |
381 | except Exception as e:
382 | # Table might not exist in test environment
383 | pytest.skip(f"Table {table} not found: {e}")
384 |
385 | except Exception as e:
386 | pytest.skip(f"Database backup test skipped: {e}")
387 |
388 | def test_configuration_backup(self):
389 | """Test configuration backup capability."""
390 |
391 | # Test that critical configuration can be backed up
392 | critical_config_files = [
393 | "alembic.ini",
394 | ".env", # Note: should not backup .env with secrets
395 | "pyproject.toml",
396 | ]
397 |
398 | project_root = Path(__file__).parent.parent
399 |
400 | for config_file in critical_config_files:
401 | config_path = project_root / config_file
402 | if config_path.exists():
403 | # Should be readable
404 | assert config_path.is_file()
405 | assert os.access(config_path, os.R_OK)
406 | else:
407 | # Some files might not exist in test environment
408 | pass
409 |
410 | def test_graceful_shutdown_capability(self, production_app):
411 | """Test graceful shutdown capability."""
412 |
413 | # Test that app can handle shutdown signals
414 | # This is more of a conceptual test since we can't actually shut down
415 |
416 | # Check that shutdown handlers are registered
417 | # This would be tested in actual deployment
418 | assert hasattr(production_app, "router")
419 | assert production_app.router is not None
420 |
421 |
422 | class TestLoadTesting:
423 | """Test system under production-like load."""
424 |
425 | @pytest.mark.skip(
426 | reason="Long-running load test - disabled to conserve CI resources"
427 | )
428 | @pytest.mark.asyncio
429 | @pytest.mark.integration
430 | async def test_concurrent_user_load(self, production_client):
431 | """Test system under concurrent user load."""
432 |
433 | # Create multiple test users
434 | test_users = []
435 | for i in range(5):
436 | user = {
437 | "email": f"loadtest{i}@example.com",
438 | "password": "LoadTest123!",
439 | "name": f"Load Test User {i}",
440 | }
441 | test_users.append(user)
442 |
443 | # Register user
444 | response = production_client.post("/auth/register", json=user)
445 | if response.status_code not in [200, 201]:
446 | pytest.skip("User registration failed in load test")
447 |
448 | # Simulate concurrent operations
449 | async def user_session(user_data):
450 | """Simulate a complete user session."""
451 | results = []
452 |
453 | # Login
454 | login_response = production_client.post(
455 | "/auth/login",
456 | json={"email": user_data["email"], "password": user_data["password"]},
457 | )
458 | results.append(("login", login_response.status_code))
459 |
460 | if login_response.status_code == 200:
461 | csrf_token = login_response.json().get("csrf_token")
462 |
463 | # Multiple API calls
464 | for _ in range(3):
465 | profile_response = production_client.get(
466 | "/user/profile", headers={"X-CSRF-Token": csrf_token}
467 | )
468 | results.append(("profile", profile_response.status_code))
469 |
470 | return results
471 |
472 | # Run concurrent sessions
473 | tasks = [user_session(user) for user in test_users]
474 | session_results = await asyncio.gather(*tasks, return_exceptions=True)
475 |
476 | # Analyze results
477 | all_results = []
478 | for result in session_results:
479 | if isinstance(result, list):
480 | all_results.extend(result)
481 |
482 | # Should have mostly successful responses
483 | success_rate = sum(
484 | 1 for op, status in all_results if status in [200, 201]
485 | ) / len(all_results)
486 | assert success_rate >= 0.8 # At least 80% success rate
487 |
488 | @pytest.mark.skip(
489 | reason="Long-running performance test - disabled to conserve CI resources"
490 | )
491 | def test_api_endpoint_performance(self, production_client):
492 | """Test API endpoint performance."""
493 |
494 | # Test key endpoints for performance
495 | endpoints_to_test = [
496 | "/health",
497 | "/",
498 | ]
499 |
500 | performance_results = {}
501 |
502 | for endpoint in endpoints_to_test:
503 | times = []
504 |
505 | for _ in range(5):
506 | start_time = time.time()
507 | response = production_client.get(endpoint)
508 | end_time = time.time()
509 |
510 | if response.status_code == 200:
511 | times.append(end_time - start_time)
512 |
513 | if times:
514 | avg_time = sum(times) / len(times)
515 | max_time = max(times)
516 | performance_results[endpoint] = {
517 | "avg_time": avg_time,
518 | "max_time": max_time,
519 | }
520 |
521 | # Performance assertions
522 | assert avg_time < 1.0 # Average response under 1 second
523 | assert max_time < 2.0 # Max response under 2 seconds
524 |
525 | @pytest.mark.skip(
526 | reason="Long-running memory test - disabled to conserve CI resources"
527 | )
528 | def test_memory_usage_stability(self, production_client):
529 | """Test memory usage stability under load."""
530 |
531 | # Make multiple requests to test for memory leaks
532 | initial_response_time = None
533 | final_response_time = None
534 |
535 | for i in range(20):
536 | start_time = time.time()
537 | response = production_client.get("/health")
538 | end_time = time.time()
539 |
540 | if response.status_code == 200:
541 | response_time = end_time - start_time
542 |
543 | if i == 0:
544 | initial_response_time = response_time
545 | elif i == 19:
546 | final_response_time = response_time
547 |
548 | # Response time should not degrade significantly (indicating memory leaks)
549 | if initial_response_time and final_response_time:
550 | degradation_ratio = final_response_time / initial_response_time
551 | assert degradation_ratio < 3.0 # Should not be 3x slower
552 |
553 |
554 | class TestProductionReadinessChecklist:
555 | """Final production readiness checklist."""
556 |
557 | def test_database_migrations_applied(self):
558 | """Test that all database migrations are applied."""
559 |
560 | try:
561 | with SessionLocal() as session:
562 | # Check that migration tables exist
563 | result = session.execute("""
564 | SELECT table_name
565 | FROM information_schema.tables
566 | WHERE table_schema = 'public'
567 | AND table_name = 'alembic_version'
568 | """)
569 |
570 | migration_table_exists = result.scalar() is not None
571 |
572 | if migration_table_exists:
573 | # Check current migration version
574 | version_result = session.execute(
575 | "SELECT version_num FROM alembic_version"
576 | )
577 | current_version = version_result.scalar()
578 |
579 | assert current_version is not None
580 | assert len(current_version) > 0
581 |
582 | except Exception as e:
583 | pytest.skip(f"Database migration check skipped: {e}")
584 |
585 | def test_security_features_enabled(self, production_settings):
586 | """Test that all security features are enabled."""
587 |
588 | # Authentication enabled
589 | assert production_settings.auth.enabled is True
590 |
591 | # Proper environment
592 | assert production_settings.environment in ["production", "staging"]
593 |
594 | def test_performance_optimizations_active(self):
595 | """Test that performance optimizations are active."""
596 |
597 | # This would test actual performance optimizations
598 | # For now, test that performance modules can be imported
599 | try:
600 | from maverick_mcp.data.performance import (
601 | query_optimizer,
602 | redis_manager,
603 | request_cache,
604 | )
605 |
606 | assert redis_manager is not None
607 | assert request_cache is not None
608 | assert query_optimizer is not None
609 |
610 | except ImportError as e:
611 | pytest.fail(f"Performance optimization modules not available: {e}")
612 |
613 | def test_monitoring_and_logging_ready(self):
614 | """Test that monitoring and logging are ready."""
615 |
616 | try:
617 | # Test logging configuration
618 | from maverick_mcp.utils.logging import get_logger
619 |
620 | logger = get_logger("production_test")
621 | assert logger is not None
622 |
623 | # Test monitoring availability
624 | from maverick_mcp.utils.monitoring import get_metrics
625 |
626 | metrics = get_metrics()
627 | assert isinstance(metrics, str)
628 |
629 | except Exception as e:
630 | pytest.skip(f"Monitoring test skipped: {e}")
631 |
632 | @pytest.mark.integration
633 | def test_final_system_integration(self, production_client):
634 | """Final system integration test."""
635 |
636 | # Test complete workflow with unique email
637 | import uuid
638 |
639 | unique_id = str(uuid.uuid4())[:8]
640 | test_user = {
641 | "email": f"final_test_{unique_id}@example.com",
642 | "password": "FinalTest123!",
643 | "name": "Final Test User",
644 | }
645 |
646 | # 1. Health check
647 | health_response = production_client.get("/health")
648 | assert health_response.status_code == 200
649 |
650 | # 2. User registration
651 | register_response = production_client.post("/auth/signup", json=test_user)
652 | assert register_response.status_code in [200, 201]
653 |
654 | # 3. User login
655 | login_response = production_client.post(
656 | "/auth/login",
657 | json={"email": test_user["email"], "password": test_user["password"]},
658 | )
659 | assert login_response.status_code == 200
660 |
661 | # Get tokens from response
662 | login_data = login_response.json()
663 | access_token = login_data.get("access_token")
664 |
665 | # If no access token in response body, it might be in cookies
666 | if not access_token:
667 | # For cookie-based auth, we just need to make sure login succeeded
668 | assert "user" in login_data or "message" in login_data
669 |
670 | # 4. Authenticated API access (with cookies)
671 | profile_response = production_client.get("/user/profile")
672 | assert profile_response.status_code == 200
673 | else:
674 | # Bearer token auth
675 | headers = {"Authorization": f"Bearer {access_token}"}
676 |
677 | # 4. Authenticated API access
678 | profile_response = production_client.get("/user/profile", headers=headers)
679 | assert profile_response.status_code == 200
680 |
681 |
682 | if __name__ == "__main__":
683 | pytest.main([__file__, "-v", "--tb=short"])
684 |
```