This is page 30 of 39. Use http://codebase.md/wshobson/maverick-mcp?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .dockerignore
├── .env.example
├── .github
│ ├── dependabot.yml
│ ├── FUNDING.yml
│ ├── ISSUE_TEMPLATE
│ │ ├── bug_report.md
│ │ ├── config.yml
│ │ ├── feature_request.md
│ │ ├── question.md
│ │ └── security_report.md
│ ├── pull_request_template.md
│ └── workflows
│ ├── claude-code-review.yml
│ └── claude.yml
├── .gitignore
├── .python-version
├── .vscode
│ ├── launch.json
│ └── settings.json
├── alembic
│ ├── env.py
│ ├── script.py.mako
│ └── versions
│ ├── 001_initial_schema.py
│ ├── 003_add_performance_indexes.py
│ ├── 006_rename_metadata_columns.py
│ ├── 008_performance_optimization_indexes.py
│ ├── 009_rename_to_supply_demand.py
│ ├── 010_self_contained_schema.py
│ ├── 011_remove_proprietary_terms.py
│ ├── 013_add_backtest_persistence_models.py
│ ├── 014_add_portfolio_models.py
│ ├── 08e3945a0c93_merge_heads.py
│ ├── 9374a5c9b679_merge_heads_for_testing.py
│ ├── abf9b9afb134_merge_multiple_heads.py
│ ├── adda6d3fd84b_merge_proprietary_terms_removal_with_.py
│ ├── e0c75b0bdadb_fix_financial_data_precision_only.py
│ ├── f0696e2cac15_add_essential_performance_indexes.py
│ └── fix_database_integrity_issues.py
├── alembic.ini
├── CLAUDE.md
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── DATABASE_SETUP.md
├── docker-compose.override.yml.example
├── docker-compose.yml
├── Dockerfile
├── docs
│ ├── api
│ │ └── backtesting.md
│ ├── BACKTESTING.md
│ ├── COST_BASIS_SPECIFICATION.md
│ ├── deep_research_agent.md
│ ├── exa_research_testing_strategy.md
│ ├── PORTFOLIO_PERSONALIZATION_PLAN.md
│ ├── PORTFOLIO.md
│ ├── SETUP_SELF_CONTAINED.md
│ └── speed_testing_framework.md
├── examples
│ ├── complete_speed_validation.py
│ ├── deep_research_integration.py
│ ├── llm_optimization_example.py
│ ├── llm_speed_demo.py
│ ├── monitoring_example.py
│ ├── parallel_research_example.py
│ ├── speed_optimization_demo.py
│ └── timeout_fix_demonstration.py
├── LICENSE
├── Makefile
├── MANIFEST.in
├── maverick_mcp
│ ├── __init__.py
│ ├── agents
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── circuit_breaker.py
│ │ ├── deep_research.py
│ │ ├── market_analysis.py
│ │ ├── optimized_research.py
│ │ ├── supervisor.py
│ │ └── technical_analysis.py
│ ├── api
│ │ ├── __init__.py
│ │ ├── api_server.py
│ │ ├── connection_manager.py
│ │ ├── dependencies
│ │ │ ├── __init__.py
│ │ │ ├── stock_analysis.py
│ │ │ └── technical_analysis.py
│ │ ├── error_handling.py
│ │ ├── inspector_compatible_sse.py
│ │ ├── inspector_sse.py
│ │ ├── middleware
│ │ │ ├── error_handling.py
│ │ │ ├── mcp_logging.py
│ │ │ ├── rate_limiting_enhanced.py
│ │ │ └── security.py
│ │ ├── openapi_config.py
│ │ ├── routers
│ │ │ ├── __init__.py
│ │ │ ├── agents.py
│ │ │ ├── backtesting.py
│ │ │ ├── data_enhanced.py
│ │ │ ├── data.py
│ │ │ ├── health_enhanced.py
│ │ │ ├── health_tools.py
│ │ │ ├── health.py
│ │ │ ├── intelligent_backtesting.py
│ │ │ ├── introspection.py
│ │ │ ├── mcp_prompts.py
│ │ │ ├── monitoring.py
│ │ │ ├── news_sentiment_enhanced.py
│ │ │ ├── performance.py
│ │ │ ├── portfolio.py
│ │ │ ├── research.py
│ │ │ ├── screening_ddd.py
│ │ │ ├── screening_parallel.py
│ │ │ ├── screening.py
│ │ │ ├── technical_ddd.py
│ │ │ ├── technical_enhanced.py
│ │ │ ├── technical.py
│ │ │ └── tool_registry.py
│ │ ├── server.py
│ │ ├── services
│ │ │ ├── __init__.py
│ │ │ ├── base_service.py
│ │ │ ├── market_service.py
│ │ │ ├── portfolio_service.py
│ │ │ ├── prompt_service.py
│ │ │ └── resource_service.py
│ │ ├── simple_sse.py
│ │ └── utils
│ │ ├── __init__.py
│ │ ├── insomnia_export.py
│ │ └── postman_export.py
│ ├── application
│ │ ├── __init__.py
│ │ ├── commands
│ │ │ └── __init__.py
│ │ ├── dto
│ │ │ ├── __init__.py
│ │ │ └── technical_analysis_dto.py
│ │ ├── queries
│ │ │ ├── __init__.py
│ │ │ └── get_technical_analysis.py
│ │ └── screening
│ │ ├── __init__.py
│ │ ├── dtos.py
│ │ └── queries.py
│ ├── backtesting
│ │ ├── __init__.py
│ │ ├── ab_testing.py
│ │ ├── analysis.py
│ │ ├── batch_processing_stub.py
│ │ ├── batch_processing.py
│ │ ├── model_manager.py
│ │ ├── optimization.py
│ │ ├── persistence.py
│ │ ├── retraining_pipeline.py
│ │ ├── strategies
│ │ │ ├── __init__.py
│ │ │ ├── base.py
│ │ │ ├── ml
│ │ │ │ ├── __init__.py
│ │ │ │ ├── adaptive.py
│ │ │ │ ├── ensemble.py
│ │ │ │ ├── feature_engineering.py
│ │ │ │ └── regime_aware.py
│ │ │ ├── ml_strategies.py
│ │ │ ├── parser.py
│ │ │ └── templates.py
│ │ ├── strategy_executor.py
│ │ ├── vectorbt_engine.py
│ │ └── visualization.py
│ ├── config
│ │ ├── __init__.py
│ │ ├── constants.py
│ │ ├── database_self_contained.py
│ │ ├── database.py
│ │ ├── llm_optimization_config.py
│ │ ├── logging_settings.py
│ │ ├── plotly_config.py
│ │ ├── security_utils.py
│ │ ├── security.py
│ │ ├── settings.py
│ │ ├── technical_constants.py
│ │ ├── tool_estimation.py
│ │ └── validation.py
│ ├── core
│ │ ├── __init__.py
│ │ ├── technical_analysis.py
│ │ └── visualization.py
│ ├── data
│ │ ├── __init__.py
│ │ ├── cache_manager.py
│ │ ├── cache.py
│ │ ├── django_adapter.py
│ │ ├── health.py
│ │ ├── models.py
│ │ ├── performance.py
│ │ ├── session_management.py
│ │ └── validation.py
│ ├── database
│ │ ├── __init__.py
│ │ ├── base.py
│ │ └── optimization.py
│ ├── dependencies.py
│ ├── domain
│ │ ├── __init__.py
│ │ ├── entities
│ │ │ ├── __init__.py
│ │ │ └── stock_analysis.py
│ │ ├── events
│ │ │ └── __init__.py
│ │ ├── portfolio.py
│ │ ├── screening
│ │ │ ├── __init__.py
│ │ │ ├── entities.py
│ │ │ ├── services.py
│ │ │ └── value_objects.py
│ │ ├── services
│ │ │ ├── __init__.py
│ │ │ └── technical_analysis_service.py
│ │ ├── stock_analysis
│ │ │ ├── __init__.py
│ │ │ └── stock_analysis_service.py
│ │ └── value_objects
│ │ ├── __init__.py
│ │ └── technical_indicators.py
│ ├── exceptions.py
│ ├── infrastructure
│ │ ├── __init__.py
│ │ ├── cache
│ │ │ └── __init__.py
│ │ ├── caching
│ │ │ ├── __init__.py
│ │ │ └── cache_management_service.py
│ │ ├── connection_manager.py
│ │ ├── data_fetching
│ │ │ ├── __init__.py
│ │ │ └── stock_data_service.py
│ │ ├── health
│ │ │ ├── __init__.py
│ │ │ └── health_checker.py
│ │ ├── persistence
│ │ │ ├── __init__.py
│ │ │ └── stock_repository.py
│ │ ├── providers
│ │ │ └── __init__.py
│ │ ├── screening
│ │ │ ├── __init__.py
│ │ │ └── repositories.py
│ │ └── sse_optimizer.py
│ ├── langchain_tools
│ │ ├── __init__.py
│ │ ├── adapters.py
│ │ └── registry.py
│ ├── logging_config.py
│ ├── memory
│ │ ├── __init__.py
│ │ └── stores.py
│ ├── monitoring
│ │ ├── __init__.py
│ │ ├── health_check.py
│ │ ├── health_monitor.py
│ │ ├── integration_example.py
│ │ ├── metrics.py
│ │ ├── middleware.py
│ │ └── status_dashboard.py
│ ├── providers
│ │ ├── __init__.py
│ │ ├── dependencies.py
│ │ ├── factories
│ │ │ ├── __init__.py
│ │ │ ├── config_factory.py
│ │ │ └── provider_factory.py
│ │ ├── implementations
│ │ │ ├── __init__.py
│ │ │ ├── cache_adapter.py
│ │ │ ├── macro_data_adapter.py
│ │ │ ├── market_data_adapter.py
│ │ │ ├── persistence_adapter.py
│ │ │ └── stock_data_adapter.py
│ │ ├── interfaces
│ │ │ ├── __init__.py
│ │ │ ├── cache.py
│ │ │ ├── config.py
│ │ │ ├── macro_data.py
│ │ │ ├── market_data.py
│ │ │ ├── persistence.py
│ │ │ └── stock_data.py
│ │ ├── llm_factory.py
│ │ ├── macro_data.py
│ │ ├── market_data.py
│ │ ├── mocks
│ │ │ ├── __init__.py
│ │ │ ├── mock_cache.py
│ │ │ ├── mock_config.py
│ │ │ ├── mock_macro_data.py
│ │ │ ├── mock_market_data.py
│ │ │ ├── mock_persistence.py
│ │ │ └── mock_stock_data.py
│ │ ├── openrouter_provider.py
│ │ ├── optimized_screening.py
│ │ ├── optimized_stock_data.py
│ │ └── stock_data.py
│ ├── README.md
│ ├── tests
│ │ ├── __init__.py
│ │ ├── README_INMEMORY_TESTS.md
│ │ ├── test_cache_debug.py
│ │ ├── test_fixes_validation.py
│ │ ├── test_in_memory_routers.py
│ │ ├── test_in_memory_server.py
│ │ ├── test_macro_data_provider.py
│ │ ├── test_mailgun_email.py
│ │ ├── test_market_calendar_caching.py
│ │ ├── test_mcp_tool_fixes_pytest.py
│ │ ├── test_mcp_tool_fixes.py
│ │ ├── test_mcp_tools.py
│ │ ├── test_models_functional.py
│ │ ├── test_server.py
│ │ ├── test_stock_data_enhanced.py
│ │ ├── test_stock_data_provider.py
│ │ └── test_technical_analysis.py
│ ├── tools
│ │ ├── __init__.py
│ │ ├── performance_monitoring.py
│ │ ├── portfolio_manager.py
│ │ ├── risk_management.py
│ │ └── sentiment_analysis.py
│ ├── utils
│ │ ├── __init__.py
│ │ ├── agent_errors.py
│ │ ├── batch_processing.py
│ │ ├── cache_warmer.py
│ │ ├── circuit_breaker_decorators.py
│ │ ├── circuit_breaker_services.py
│ │ ├── circuit_breaker.py
│ │ ├── data_chunking.py
│ │ ├── database_monitoring.py
│ │ ├── debug_utils.py
│ │ ├── fallback_strategies.py
│ │ ├── llm_optimization.py
│ │ ├── logging_example.py
│ │ ├── logging_init.py
│ │ ├── logging.py
│ │ ├── mcp_logging.py
│ │ ├── memory_profiler.py
│ │ ├── monitoring_middleware.py
│ │ ├── monitoring.py
│ │ ├── orchestration_logging.py
│ │ ├── parallel_research.py
│ │ ├── parallel_screening.py
│ │ ├── quick_cache.py
│ │ ├── resource_manager.py
│ │ ├── shutdown.py
│ │ ├── stock_helpers.py
│ │ ├── structured_logger.py
│ │ ├── tool_monitoring.py
│ │ ├── tracing.py
│ │ └── yfinance_pool.py
│ ├── validation
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── data.py
│ │ ├── middleware.py
│ │ ├── portfolio.py
│ │ ├── responses.py
│ │ ├── screening.py
│ │ └── technical.py
│ └── workflows
│ ├── __init__.py
│ ├── agents
│ │ ├── __init__.py
│ │ ├── market_analyzer.py
│ │ ├── optimizer_agent.py
│ │ ├── strategy_selector.py
│ │ └── validator_agent.py
│ ├── backtesting_workflow.py
│ └── state.py
├── PLANS.md
├── pyproject.toml
├── pyrightconfig.json
├── README.md
├── scripts
│ ├── dev.sh
│ ├── INSTALLATION_GUIDE.md
│ ├── load_example.py
│ ├── load_market_data.py
│ ├── load_tiingo_data.py
│ ├── migrate_db.py
│ ├── README_TIINGO_LOADER.md
│ ├── requirements_tiingo.txt
│ ├── run_stock_screening.py
│ ├── run-migrations.sh
│ ├── seed_db.py
│ ├── seed_sp500.py
│ ├── setup_database.sh
│ ├── setup_self_contained.py
│ ├── setup_sp500_database.sh
│ ├── test_seeded_data.py
│ ├── test_tiingo_loader.py
│ ├── tiingo_config.py
│ └── validate_setup.py
├── SECURITY.md
├── server.json
├── setup.py
├── tests
│ ├── __init__.py
│ ├── conftest.py
│ ├── core
│ │ └── test_technical_analysis.py
│ ├── data
│ │ └── test_portfolio_models.py
│ ├── domain
│ │ ├── conftest.py
│ │ ├── test_portfolio_entities.py
│ │ └── test_technical_analysis_service.py
│ ├── fixtures
│ │ └── orchestration_fixtures.py
│ ├── integration
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── README.md
│ │ ├── run_integration_tests.sh
│ │ ├── test_api_technical.py
│ │ ├── test_chaos_engineering.py
│ │ ├── test_config_management.py
│ │ ├── test_full_backtest_workflow_advanced.py
│ │ ├── test_full_backtest_workflow.py
│ │ ├── test_high_volume.py
│ │ ├── test_mcp_tools.py
│ │ ├── test_orchestration_complete.py
│ │ ├── test_portfolio_persistence.py
│ │ ├── test_redis_cache.py
│ │ ├── test_security_integration.py.disabled
│ │ └── vcr_setup.py
│ ├── performance
│ │ ├── __init__.py
│ │ ├── test_benchmarks.py
│ │ ├── test_load.py
│ │ ├── test_profiling.py
│ │ └── test_stress.py
│ ├── providers
│ │ └── test_stock_data_simple.py
│ ├── README.md
│ ├── test_agents_router_mcp.py
│ ├── test_backtest_persistence.py
│ ├── test_cache_management_service.py
│ ├── test_cache_serialization.py
│ ├── test_circuit_breaker.py
│ ├── test_database_pool_config_simple.py
│ ├── test_database_pool_config.py
│ ├── test_deep_research_functional.py
│ ├── test_deep_research_integration.py
│ ├── test_deep_research_parallel_execution.py
│ ├── test_error_handling.py
│ ├── test_event_loop_integrity.py
│ ├── test_exa_research_integration.py
│ ├── test_exception_hierarchy.py
│ ├── test_financial_search.py
│ ├── test_graceful_shutdown.py
│ ├── test_integration_simple.py
│ ├── test_langgraph_workflow.py
│ ├── test_market_data_async.py
│ ├── test_market_data_simple.py
│ ├── test_mcp_orchestration_functional.py
│ ├── test_ml_strategies.py
│ ├── test_optimized_research_agent.py
│ ├── test_orchestration_integration.py
│ ├── test_orchestration_logging.py
│ ├── test_orchestration_tools_simple.py
│ ├── test_parallel_research_integration.py
│ ├── test_parallel_research_orchestrator.py
│ ├── test_parallel_research_performance.py
│ ├── test_performance_optimizations.py
│ ├── test_production_validation.py
│ ├── test_provider_architecture.py
│ ├── test_rate_limiting_enhanced.py
│ ├── test_runner_validation.py
│ ├── test_security_comprehensive.py.disabled
│ ├── test_security_cors.py
│ ├── test_security_enhancements.py.disabled
│ ├── test_security_headers.py
│ ├── test_security_penetration.py
│ ├── test_session_management.py
│ ├── test_speed_optimization_validation.py
│ ├── test_stock_analysis_dependencies.py
│ ├── test_stock_analysis_service.py
│ ├── test_stock_data_fetching_service.py
│ ├── test_supervisor_agent.py
│ ├── test_supervisor_functional.py
│ ├── test_tool_estimation_config.py
│ ├── test_visualization.py
│ └── utils
│ ├── test_agent_errors.py
│ ├── test_logging.py
│ ├── test_parallel_screening.py
│ └── test_quick_cache.py
├── tools
│ ├── check_orchestration_config.py
│ ├── experiments
│ │ ├── validation_examples.py
│ │ └── validation_fixed.py
│ ├── fast_dev.sh
│ ├── hot_reload.py
│ ├── quick_test.py
│ └── templates
│ ├── new_router_template.py
│ ├── new_tool_template.py
│ ├── screening_strategy_template.py
│ └── test_template.py
└── uv.lock
```
# Files
--------------------------------------------------------------------------------
/tests/test_orchestration_logging.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Comprehensive test suite for OrchestrationLogger functionality.
3 |
4 | This test suite covers:
5 | - OrchestrationLogger initialization and configuration
6 | - Context tracking and performance metrics
7 | - Visual indicators and structured output formatting
8 | - Parallel execution logging
9 | - Method call decorators and context managers
10 | - Resource usage monitoring
11 | - Error handling and fallback logging
12 | """
13 |
14 | import asyncio
15 | import logging
16 | import time
17 | from unittest.mock import Mock, patch
18 |
19 | import pytest
20 |
21 | from maverick_mcp.utils.orchestration_logging import (
22 | LogColors,
23 | OrchestrationLogger,
24 | get_orchestration_logger,
25 | log_agent_execution,
26 | log_fallback_trigger,
27 | log_method_call,
28 | log_parallel_execution,
29 | log_performance_metrics,
30 | log_resource_usage,
31 | log_synthesis_operation,
32 | log_tool_invocation,
33 | )
34 |
35 |
36 | class TestLogColors:
37 | """Test LogColors utility class."""
38 |
39 | def test_color_constants(self):
40 | """Test that color constants are defined."""
41 | assert hasattr(LogColors, "HEADER")
42 | assert hasattr(LogColors, "OKBLUE")
43 | assert hasattr(LogColors, "OKCYAN")
44 | assert hasattr(LogColors, "OKGREEN")
45 | assert hasattr(LogColors, "WARNING")
46 | assert hasattr(LogColors, "FAIL")
47 | assert hasattr(LogColors, "ENDC")
48 | assert hasattr(LogColors, "BOLD")
49 | assert hasattr(LogColors, "UNDERLINE")
50 |
51 | # Verify they contain ANSI escape sequences
52 | assert LogColors.HEADER.startswith("\033[")
53 | assert LogColors.ENDC == "\033[0m"
54 |
55 |
56 | class TestOrchestrationLogger:
57 | """Test OrchestrationLogger main functionality."""
58 |
59 | def test_logger_initialization(self):
60 | """Test OrchestrationLogger initialization."""
61 | logger = OrchestrationLogger("TestComponent")
62 |
63 | assert logger.component_name == "TestComponent"
64 | assert logger.request_id is None
65 | assert logger.session_context == {}
66 | assert isinstance(logger.logger, logging.Logger)
67 | assert logger.logger.name == "maverick_mcp.orchestration.TestComponent"
68 |
69 | def test_set_request_context(self):
70 | """Test setting request context."""
71 | logger = OrchestrationLogger("TestComponent")
72 |
73 | # Test with explicit request_id
74 | logger.set_request_context(
75 | request_id="req_123", session_id="session_456", custom_param="value"
76 | )
77 |
78 | assert logger.request_id == "req_123"
79 | assert logger.session_context["session_id"] == "session_456"
80 | assert logger.session_context["request_id"] == "req_123"
81 | assert logger.session_context["custom_param"] == "value"
82 |
83 | def test_set_request_context_auto_id(self):
84 | """Test auto-generation of request ID."""
85 | logger = OrchestrationLogger("TestComponent")
86 |
87 | logger.set_request_context(session_id="session_789")
88 |
89 | assert logger.request_id is not None
90 | assert len(logger.request_id) == 8 # UUID truncated to 8 chars
91 | assert logger.session_context["session_id"] == "session_789"
92 | assert logger.session_context["request_id"] == logger.request_id
93 |
94 | def test_format_message_with_context(self):
95 | """Test message formatting with context."""
96 | logger = OrchestrationLogger("TestComponent")
97 | logger.set_request_context(request_id="req_123", session_id="session_456")
98 |
99 | formatted = logger._format_message(
100 | "INFO", "Test message", param1="value1", param2=42
101 | )
102 |
103 | assert "TestComponent" in formatted
104 | assert "req:req_123" in formatted
105 | assert "session:session_456" in formatted
106 | assert "Test message" in formatted
107 | assert "param1:value1" in formatted
108 | assert "param2:42" in formatted
109 |
110 | def test_format_message_without_context(self):
111 | """Test message formatting without context."""
112 | logger = OrchestrationLogger("TestComponent")
113 |
114 | formatted = logger._format_message("WARNING", "Warning message", error="test")
115 |
116 | assert "TestComponent" in formatted
117 | assert "Warning message" in formatted
118 | assert "error:test" in formatted
119 | # Should not contain context brackets when no context
120 | assert "req:" not in formatted
121 | assert "session:" not in formatted
122 |
123 | def test_format_message_color_coding(self):
124 | """Test color coding in message formatting."""
125 | logger = OrchestrationLogger("TestComponent")
126 |
127 | debug_msg = logger._format_message("DEBUG", "Debug message")
128 | info_msg = logger._format_message("INFO", "Info message")
129 | warning_msg = logger._format_message("WARNING", "Warning message")
130 | error_msg = logger._format_message("ERROR", "Error message")
131 |
132 | assert LogColors.OKCYAN in debug_msg
133 | assert LogColors.OKGREEN in info_msg
134 | assert LogColors.WARNING in warning_msg
135 | assert LogColors.FAIL in error_msg
136 |
137 | # All should end with reset color
138 | assert LogColors.ENDC in debug_msg
139 | assert LogColors.ENDC in info_msg
140 | assert LogColors.ENDC in warning_msg
141 | assert LogColors.ENDC in error_msg
142 |
143 | def test_logging_methods(self):
144 | """Test all logging level methods."""
145 | logger = OrchestrationLogger("TestComponent")
146 |
147 | with (
148 | patch.object(logger.logger, "debug") as mock_debug,
149 | patch.object(logger.logger, "info") as mock_info,
150 | patch.object(logger.logger, "warning") as mock_warning,
151 | patch.object(logger.logger, "error") as mock_error,
152 | ):
153 | logger.debug("Debug message", param="debug")
154 | logger.info("Info message", param="info")
155 | logger.warning("Warning message", param="warning")
156 | logger.error("Error message", param="error")
157 |
158 | mock_debug.assert_called_once()
159 | mock_info.assert_called_once()
160 | mock_warning.assert_called_once()
161 | mock_error.assert_called_once()
162 |
163 | def test_none_value_filtering(self):
164 | """Test filtering of None values in message formatting."""
165 | logger = OrchestrationLogger("TestComponent")
166 |
167 | formatted = logger._format_message(
168 | "INFO",
169 | "Test message",
170 | param1="value1",
171 | param2=None, # Should be filtered out
172 | param3="value3",
173 | )
174 |
175 | assert "param1:value1" in formatted
176 | assert "param2:None" not in formatted
177 | assert "param3:value3" in formatted
178 |
179 |
180 | class TestGlobalLoggerRegistry:
181 | """Test global logger registry functionality."""
182 |
183 | def test_get_orchestration_logger_creation(self):
184 | """Test creation of new orchestration logger."""
185 | logger = get_orchestration_logger("NewComponent")
186 |
187 | assert isinstance(logger, OrchestrationLogger)
188 | assert logger.component_name == "NewComponent"
189 |
190 | def test_get_orchestration_logger_reuse(self):
191 | """Test reuse of existing orchestration logger."""
192 | logger1 = get_orchestration_logger("ReuseComponent")
193 | logger2 = get_orchestration_logger("ReuseComponent")
194 |
195 | assert logger1 is logger2 # Should be the same instance
196 |
197 | def test_multiple_component_loggers(self):
198 | """Test multiple independent component loggers."""
199 | logger_a = get_orchestration_logger("ComponentA")
200 | logger_b = get_orchestration_logger("ComponentB")
201 |
202 | assert logger_a is not logger_b
203 | assert logger_a.component_name == "ComponentA"
204 | assert logger_b.component_name == "ComponentB"
205 |
206 |
207 | class TestLogMethodCallDecorator:
208 | """Test log_method_call decorator functionality."""
209 |
210 | @pytest.fixture
211 | def sample_class(self):
212 | """Create sample class for decorator testing."""
213 |
214 | class SampleClass:
215 | def __init__(self):
216 | self.name = "SampleClass"
217 |
218 | @log_method_call(component="TestComponent")
219 | async def async_method(self, param1: str, param2: int = 10):
220 | await asyncio.sleep(0.01)
221 | return f"result_{param1}_{param2}"
222 |
223 | @log_method_call(component="TestComponent", include_params=False)
224 | async def async_method_no_params(self):
225 | return "no_params_result"
226 |
227 | @log_method_call(component="TestComponent", include_timing=False)
228 | async def async_method_no_timing(self):
229 | return "no_timing_result"
230 |
231 | @log_method_call()
232 | def sync_method(self, value: str):
233 | return f"sync_{value}"
234 |
235 | return SampleClass
236 |
237 | @pytest.mark.asyncio
238 | async def test_async_method_decoration_success(self, sample_class):
239 | """Test successful async method decoration."""
240 | instance = sample_class()
241 |
242 | with patch(
243 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
244 | ) as mock_get_logger:
245 | mock_logger = Mock()
246 | mock_get_logger.return_value = mock_logger
247 |
248 | result = await instance.async_method("test", param2=20)
249 |
250 | assert result == "result_test_20"
251 |
252 | # Verify logging calls
253 | assert mock_logger.info.call_count == 2 # Start and success
254 | start_call = mock_logger.info.call_args_list[0][0][0]
255 | success_call = mock_logger.info.call_args_list[1][0][0]
256 |
257 | assert "🚀 START async_method" in start_call
258 | assert "params:" in start_call
259 | assert "✅ SUCCESS async_method" in success_call
260 | assert "duration:" in success_call
261 |
262 | @pytest.mark.asyncio
263 | async def test_async_method_decoration_no_params(self, sample_class):
264 | """Test async method decoration without parameter logging."""
265 | instance = sample_class()
266 |
267 | with patch(
268 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
269 | ) as mock_get_logger:
270 | mock_logger = Mock()
271 | mock_get_logger.return_value = mock_logger
272 |
273 | await instance.async_method_no_params()
274 |
275 | start_call = mock_logger.info.call_args_list[0][0][0]
276 | assert "params:" not in start_call
277 |
278 | @pytest.mark.asyncio
279 | async def test_async_method_decoration_no_timing(self, sample_class):
280 | """Test async method decoration without timing."""
281 | instance = sample_class()
282 |
283 | with patch(
284 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
285 | ) as mock_get_logger:
286 | mock_logger = Mock()
287 | mock_get_logger.return_value = mock_logger
288 |
289 | await instance.async_method_no_timing()
290 |
291 | success_call = mock_logger.info.call_args_list[1][0][0]
292 | assert "duration:" not in success_call
293 |
294 | @pytest.mark.asyncio
295 | async def test_async_method_decoration_error(self, sample_class):
296 | """Test async method decoration with error handling."""
297 |
298 | class ErrorClass:
299 | @log_method_call(component="ErrorComponent")
300 | async def failing_method(self):
301 | raise ValueError("Test error")
302 |
303 | instance = ErrorClass()
304 |
305 | with patch(
306 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
307 | ) as mock_get_logger:
308 | mock_logger = Mock()
309 | mock_get_logger.return_value = mock_logger
310 |
311 | with pytest.raises(ValueError, match="Test error"):
312 | await instance.failing_method()
313 |
314 | # Should log error
315 | assert mock_logger.error.called
316 | error_call = mock_logger.error.call_args[0][0]
317 | assert "❌ ERROR failing_method" in error_call
318 | assert "error: Test error" in error_call
319 |
320 | def test_sync_method_decoration(self, sample_class):
321 | """Test synchronous method decoration."""
322 | instance = sample_class()
323 |
324 | with patch(
325 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
326 | ) as mock_get_logger:
327 | mock_logger = Mock()
328 | mock_get_logger.return_value = mock_logger
329 |
330 | result = instance.sync_method("test_value")
331 |
332 | assert result == "sync_test_value"
333 |
334 | # Should log start and success
335 | assert mock_logger.info.call_count == 2
336 | assert "🚀 START sync_method" in mock_logger.info.call_args_list[0][0][0]
337 | assert "✅ SUCCESS sync_method" in mock_logger.info.call_args_list[1][0][0]
338 |
339 | def test_component_name_inference(self):
340 | """Test automatic component name inference."""
341 |
342 | class InferenceTest:
343 | @log_method_call()
344 | def test_method(self):
345 | return "test"
346 |
347 | instance = InferenceTest()
348 |
349 | with patch(
350 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
351 | ) as mock_get_logger:
352 | mock_logger = Mock()
353 | mock_get_logger.return_value = mock_logger
354 |
355 | instance.test_method()
356 |
357 | # Should infer component name from class
358 | mock_get_logger.assert_called_with("InferenceTest")
359 |
360 | def test_result_summary_extraction(self, sample_class):
361 | """Test extraction of result summaries for logging."""
362 |
363 | class ResultClass:
364 | @log_method_call(component="ResultComponent")
365 | async def method_with_result_info(self):
366 | return {
367 | "execution_mode": "parallel",
368 | "research_confidence": 0.85,
369 | "parallel_execution_stats": {
370 | "successful_tasks": 3,
371 | "total_tasks": 4,
372 | },
373 | }
374 |
375 | instance = ResultClass()
376 |
377 | with patch(
378 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
379 | ) as mock_get_logger:
380 | mock_logger = Mock()
381 | mock_get_logger.return_value = mock_logger
382 |
383 | asyncio.run(instance.method_with_result_info())
384 |
385 | success_call = mock_logger.info.call_args_list[1][0][0]
386 | assert "mode: parallel" in success_call
387 | assert "confidence: 0.85" in success_call
388 | assert "tasks: 3/4" in success_call
389 |
390 |
391 | class TestContextManagers:
392 | """Test context manager utilities."""
393 |
394 | def test_log_parallel_execution_success(self):
395 | """Test log_parallel_execution context manager success case."""
396 | with patch(
397 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
398 | ) as mock_get_logger:
399 | mock_logger = Mock()
400 | mock_get_logger.return_value = mock_logger
401 |
402 | with log_parallel_execution("TestComponent", "test operation", 3) as logger:
403 | assert logger == mock_logger
404 | time.sleep(0.01) # Simulate work
405 |
406 | # Should log start and success
407 | assert mock_logger.info.call_count == 2
408 | start_call = mock_logger.info.call_args_list[0][0][0]
409 | success_call = mock_logger.info.call_args_list[1][0][0]
410 |
411 | assert "🔄 PARALLEL_START test operation" in start_call
412 | assert "tasks: 3" in start_call
413 | assert "🎯 PARALLEL_SUCCESS test operation" in success_call
414 | assert "duration:" in success_call
415 |
416 | def test_log_parallel_execution_error(self):
417 | """Test log_parallel_execution context manager error case."""
418 | with patch(
419 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
420 | ) as mock_get_logger:
421 | mock_logger = Mock()
422 | mock_get_logger.return_value = mock_logger
423 |
424 | with pytest.raises(ValueError, match="Test error"):
425 | with log_parallel_execution("TestComponent", "failing operation", 2):
426 | raise ValueError("Test error")
427 |
428 | # Should log start and error
429 | assert mock_logger.info.call_count == 1 # Only start
430 | assert mock_logger.error.call_count == 1
431 |
432 | start_call = mock_logger.info.call_args[0][0]
433 | error_call = mock_logger.error.call_args[0][0]
434 |
435 | assert "🔄 PARALLEL_START failing operation" in start_call
436 | assert "💥 PARALLEL_ERROR failing operation" in error_call
437 | assert "error: Test error" in error_call
438 |
439 | def test_log_agent_execution_success(self):
440 | """Test log_agent_execution context manager success case."""
441 | with patch(
442 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
443 | ) as mock_get_logger:
444 | mock_logger = Mock()
445 | mock_get_logger.return_value = mock_logger
446 |
447 | with log_agent_execution(
448 | "fundamental", "task_123", ["earnings", "valuation"]
449 | ) as logger:
450 | assert logger == mock_logger
451 | time.sleep(0.01)
452 |
453 | # Should log start and success
454 | assert mock_logger.info.call_count == 2
455 | start_call = mock_logger.info.call_args_list[0][0][0]
456 | success_call = mock_logger.info.call_args_list[1][0][0]
457 |
458 | assert "🤖 AGENT_START task_123" in start_call
459 | assert "focus: ['earnings', 'valuation']" in start_call
460 | assert "🎉 AGENT_SUCCESS task_123" in success_call
461 |
462 | def test_log_agent_execution_without_focus(self):
463 | """Test log_agent_execution without focus areas."""
464 | with patch(
465 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
466 | ) as mock_get_logger:
467 | mock_logger = Mock()
468 | mock_get_logger.return_value = mock_logger
469 |
470 | with log_agent_execution("sentiment", "task_456"):
471 | pass
472 |
473 | start_call = mock_logger.info.call_args_list[0][0][0]
474 | assert "focus:" not in start_call
475 |
476 | def test_log_agent_execution_error(self):
477 | """Test log_agent_execution context manager error case."""
478 | with patch(
479 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
480 | ) as mock_get_logger:
481 | mock_logger = Mock()
482 | mock_get_logger.return_value = mock_logger
483 |
484 | with pytest.raises(RuntimeError, match="Agent failed"):
485 | with log_agent_execution("technical", "task_789"):
486 | raise RuntimeError("Agent failed")
487 |
488 | # Should log start and error
489 | error_call = mock_logger.error.call_args[0][0]
490 | assert "🔥 AGENT_ERROR task_789" in error_call
491 | assert "error: Agent failed" in error_call
492 |
493 |
494 | class TestUtilityLoggingFunctions:
495 | """Test utility logging functions."""
496 |
497 | def test_log_tool_invocation_basic(self):
498 | """Test basic tool invocation logging."""
499 | with patch(
500 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
501 | ) as mock_get_logger:
502 | mock_logger = Mock()
503 | mock_get_logger.return_value = mock_logger
504 |
505 | log_tool_invocation("test_tool")
506 |
507 | mock_logger.info.assert_called_once()
508 | call_arg = mock_logger.info.call_args[0][0]
509 | assert "🔧 TOOL_INVOKE test_tool" in call_arg
510 |
511 | def test_log_tool_invocation_with_request_data(self):
512 | """Test tool invocation logging with request data."""
513 | with patch(
514 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
515 | ) as mock_get_logger:
516 | mock_logger = Mock()
517 | mock_get_logger.return_value = mock_logger
518 |
519 | request_data = {
520 | "query": "This is a test query that is longer than 50 characters to test truncation",
521 | "research_scope": "comprehensive",
522 | "persona": "moderate",
523 | }
524 |
525 | log_tool_invocation("research_tool", request_data)
526 |
527 | call_arg = mock_logger.info.call_args[0][0]
528 | assert "🔧 TOOL_INVOKE research_tool" in call_arg
529 | assert (
530 | "query: 'This is a test query that is longer than 50 charac...'"
531 | in call_arg
532 | )
533 | assert "scope: comprehensive" in call_arg
534 | assert "persona: moderate" in call_arg
535 |
536 | def test_log_synthesis_operation(self):
537 | """Test synthesis operation logging."""
538 | with patch(
539 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
540 | ) as mock_get_logger:
541 | mock_logger = Mock()
542 | mock_get_logger.return_value = mock_logger
543 |
544 | log_synthesis_operation(
545 | "parallel_research", 5, "Combined insights from multiple agents"
546 | )
547 |
548 | call_arg = mock_logger.info.call_args[0][0]
549 | assert "🧠 SYNTHESIS parallel_research" in call_arg
550 | assert "inputs: 5" in call_arg
551 | assert "output: Combined insights from multiple agents" in call_arg
552 |
553 | def test_log_synthesis_operation_without_output(self):
554 | """Test synthesis operation logging without output summary."""
555 | with patch(
556 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
557 | ) as mock_get_logger:
558 | mock_logger = Mock()
559 | mock_get_logger.return_value = mock_logger
560 |
561 | log_synthesis_operation("basic_synthesis", 3)
562 |
563 | call_arg = mock_logger.info.call_args[0][0]
564 | assert "🧠 SYNTHESIS basic_synthesis" in call_arg
565 | assert "inputs: 3" in call_arg
566 | assert "output:" not in call_arg
567 |
568 | def test_log_fallback_trigger(self):
569 | """Test fallback trigger logging."""
570 | with patch(
571 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
572 | ) as mock_get_logger:
573 | mock_logger = Mock()
574 | mock_get_logger.return_value = mock_logger
575 |
576 | log_fallback_trigger(
577 | "ParallelOrchestrator", "API timeout", "switch to sequential"
578 | )
579 |
580 | mock_logger.warning.assert_called_once()
581 | call_arg = mock_logger.warning.call_args[0][0]
582 | assert "⚠️ FALLBACK_TRIGGER API timeout" in call_arg
583 | assert "action: switch to sequential" in call_arg
584 |
585 | def test_log_performance_metrics(self):
586 | """Test performance metrics logging."""
587 | with patch(
588 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
589 | ) as mock_get_logger:
590 | mock_logger = Mock()
591 | mock_get_logger.return_value = mock_logger
592 |
593 | metrics = {
594 | "total_tasks": 5,
595 | "successful_tasks": 4,
596 | "failed_tasks": 1,
597 | "parallel_efficiency": 2.3,
598 | "total_duration": 1.5,
599 | }
600 |
601 | log_performance_metrics("TestComponent", metrics)
602 |
603 | call_arg = mock_logger.info.call_args[0][0]
604 | assert "📊 PERFORMANCE_METRICS" in call_arg
605 | assert "total_tasks: 5" in call_arg
606 | assert "successful_tasks: 4" in call_arg
607 | assert "parallel_efficiency: 2.3" in call_arg
608 |
609 | def test_log_resource_usage_complete(self):
610 | """Test resource usage logging with all parameters."""
611 | with patch(
612 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
613 | ) as mock_get_logger:
614 | mock_logger = Mock()
615 | mock_get_logger.return_value = mock_logger
616 |
617 | log_resource_usage(
618 | "ResourceComponent", api_calls=15, cache_hits=8, memory_mb=45.7
619 | )
620 |
621 | call_arg = mock_logger.info.call_args[0][0]
622 | assert "📈 RESOURCE_USAGE" in call_arg
623 | assert "api_calls: 15" in call_arg
624 | assert "cache_hits: 8" in call_arg
625 | assert "memory_mb: 45.7" in call_arg
626 |
627 | def test_log_resource_usage_partial(self):
628 | """Test resource usage logging with partial parameters."""
629 | with patch(
630 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
631 | ) as mock_get_logger:
632 | mock_logger = Mock()
633 | mock_get_logger.return_value = mock_logger
634 |
635 | log_resource_usage("ResourceComponent", api_calls=10, cache_hits=None)
636 |
637 | call_arg = mock_logger.info.call_args[0][0]
638 | assert "api_calls: 10" in call_arg
639 | assert "cache_hits" not in call_arg
640 | assert "memory_mb" not in call_arg
641 |
642 | def test_log_resource_usage_no_params(self):
643 | """Test resource usage logging with no valid parameters."""
644 | with patch(
645 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
646 | ) as mock_get_logger:
647 | mock_logger = Mock()
648 | mock_get_logger.return_value = mock_logger
649 |
650 | log_resource_usage(
651 | "ResourceComponent", api_calls=None, cache_hits=None, memory_mb=None
652 | )
653 |
654 | # Should not call logger if no valid parameters
655 | mock_logger.info.assert_not_called()
656 |
657 |
658 | class TestIntegratedLoggingScenarios:
659 | """Test integrated logging scenarios."""
660 |
661 | @pytest.mark.asyncio
662 | async def test_complete_parallel_research_logging(self):
663 | """Test complete parallel research logging scenario."""
664 | with patch(
665 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
666 | ) as mock_get_logger:
667 | mock_logger = Mock()
668 | mock_get_logger.return_value = mock_logger
669 |
670 | # Simulate complete parallel research workflow
671 | class MockResearchAgent:
672 | @log_method_call(component="ParallelOrchestrator")
673 | async def execute_parallel_research(self, topic: str, session_id: str):
674 | # Set context for this research session
675 | orchestration_logger = get_orchestration_logger(
676 | "ParallelOrchestrator"
677 | )
678 | orchestration_logger.set_request_context(
679 | session_id=session_id, research_topic=topic[:50], task_count=3
680 | )
681 |
682 | # Log tool invocation
683 | log_tool_invocation(
684 | "deep_research",
685 | {"query": topic, "research_scope": "comprehensive"},
686 | )
687 |
688 | # Execute parallel tasks
689 | with log_parallel_execution(
690 | "ParallelOrchestrator", "research execution", 3
691 | ):
692 | # Simulate parallel agent executions
693 | for i, agent_type in enumerate(
694 | ["fundamental", "sentiment", "technical"]
695 | ):
696 | with log_agent_execution(
697 | agent_type, f"task_{i}", ["focus1", "focus2"]
698 | ):
699 | await asyncio.sleep(0.01) # Simulate work
700 |
701 | # Log synthesis
702 | log_synthesis_operation(
703 | "parallel_research_synthesis", 3, "Comprehensive analysis"
704 | )
705 |
706 | # Log performance metrics
707 | log_performance_metrics(
708 | "ParallelOrchestrator",
709 | {
710 | "successful_tasks": 3,
711 | "failed_tasks": 0,
712 | "parallel_efficiency": 2.5,
713 | },
714 | )
715 |
716 | # Log resource usage
717 | log_resource_usage(
718 | "ParallelOrchestrator", api_calls=15, cache_hits=5
719 | )
720 |
721 | return {
722 | "status": "success",
723 | "execution_mode": "parallel",
724 | "research_confidence": 0.85,
725 | }
726 |
727 | agent = MockResearchAgent()
728 | result = await agent.execute_parallel_research(
729 | topic="Apple Inc comprehensive analysis",
730 | session_id="integrated_test_123",
731 | )
732 |
733 | # Verify comprehensive logging occurred
734 | assert mock_logger.info.call_count >= 8 # Multiple info logs expected
735 | assert result["status"] == "success"
736 |
737 | def test_logging_component_isolation(self):
738 | """Test that different components maintain separate logging contexts."""
739 | with patch(
740 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
741 | ) as mock_get_logger:
742 | mock_logger_a = Mock()
743 | mock_logger_b = Mock()
744 |
745 | # Mock different loggers for different components
746 | def get_logger_side_effect(component_name):
747 | if component_name == "ComponentA":
748 | return mock_logger_a
749 | elif component_name == "ComponentB":
750 | return mock_logger_b
751 | else:
752 | return Mock()
753 |
754 | mock_get_logger.side_effect = get_logger_side_effect
755 |
756 | # Component A operations
757 | log_performance_metrics("ComponentA", {"metric_a": 1})
758 | log_resource_usage("ComponentA", api_calls=5)
759 |
760 | # Component B operations
761 | log_performance_metrics("ComponentB", {"metric_b": 2})
762 | log_fallback_trigger("ComponentB", "test reason", "test action")
763 |
764 | # Verify isolation
765 | assert mock_logger_a.info.call_count == 2 # Performance + resource
766 | assert mock_logger_b.info.call_count == 1 # Performance only
767 | assert mock_logger_b.warning.call_count == 1 # Fallback trigger
768 |
769 | @pytest.mark.asyncio
770 | async def test_error_propagation_with_logging(self):
771 | """Test that errors are properly logged and propagated."""
772 | with patch(
773 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
774 | ) as mock_get_logger:
775 | mock_logger = Mock()
776 | mock_get_logger.return_value = mock_logger
777 |
778 | class ErrorComponent:
779 | @log_method_call(component="ErrorComponent")
780 | async def failing_operation(self):
781 | with log_parallel_execution("ErrorComponent", "failing task", 1):
782 | with log_agent_execution("test_agent", "failing_task"):
783 | raise RuntimeError("Simulated failure")
784 |
785 | component = ErrorComponent()
786 |
787 | # Should properly propagate the error while logging it
788 | with pytest.raises(RuntimeError, match="Simulated failure"):
789 | await component.failing_operation()
790 |
791 | # Verify error was logged at multiple levels
792 | assert (
793 | mock_logger.error.call_count >= 2
794 | ) # Method and context manager errors
795 |
796 | def test_performance_timing_accuracy(self):
797 | """Test timing accuracy in logging decorators and context managers."""
798 | with patch(
799 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
800 | ) as mock_get_logger:
801 | mock_logger = Mock()
802 | mock_get_logger.return_value = mock_logger
803 |
804 | @log_method_call(component="TimingTest")
805 | def timed_function():
806 | time.sleep(0.1) # Sleep for ~100ms
807 | return "completed"
808 |
809 | result = timed_function()
810 |
811 | assert result == "completed"
812 |
813 | # Check that timing was logged
814 | success_call = mock_logger.info.call_args_list[1][0][0]
815 | assert "duration:" in success_call
816 |
817 | # Extract duration (rough check - timing can be imprecise in tests)
818 | duration_part = [
819 | part for part in success_call.split() if "duration:" in part
820 | ][0]
821 | duration_value = float(duration_part.split(":")[-1].replace("s", ""))
822 | assert (
823 | 0.05 <= duration_value <= 0.5
824 | ) # Should be around 0.1s with some tolerance
825 |
826 |
827 | class TestLoggingUnderLoad:
828 | """Test logging behavior under various load conditions."""
829 |
830 | @pytest.mark.asyncio
831 | async def test_concurrent_logging_safety(self):
832 | """Test that concurrent logging operations are safe."""
833 | with patch(
834 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
835 | ) as mock_get_logger:
836 | mock_logger = Mock()
837 | mock_get_logger.return_value = mock_logger
838 |
839 | @log_method_call(component="ConcurrentTest")
840 | async def concurrent_task(task_id: int):
841 | with log_agent_execution("test_agent", f"task_{task_id}"):
842 | await asyncio.sleep(0.01)
843 | return f"result_{task_id}"
844 |
845 | # Run multiple tasks concurrently
846 | tasks = [concurrent_task(i) for i in range(5)]
847 | results = await asyncio.gather(*tasks)
848 |
849 | # Verify all tasks completed
850 | assert len(results) == 5
851 | assert all("result_" in result for result in results)
852 |
853 | # Logging should have occurred for all tasks
854 | assert mock_logger.info.call_count >= 10 # At least 2 per task
855 |
856 | def test_high_frequency_logging(self):
857 | """Test logging performance under high frequency operations."""
858 | with patch(
859 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
860 | ) as mock_get_logger:
861 | mock_logger = Mock()
862 | mock_get_logger.return_value = mock_logger
863 |
864 | # Perform many logging operations quickly
865 | start_time = time.time()
866 |
867 | for i in range(100):
868 | log_performance_metrics(
869 | f"Component_{i % 5}", {"operation_id": i, "timestamp": time.time()}
870 | )
871 |
872 | end_time = time.time()
873 |
874 | # Should complete quickly
875 | assert (end_time - start_time) < 1.0 # Should take less than 1 second
876 |
877 | # All operations should have been logged
878 | assert mock_logger.info.call_count == 100
879 |
880 | @pytest.mark.asyncio
881 | async def test_memory_usage_tracking(self):
882 | """Test that logging doesn't consume excessive memory."""
883 | import gc
884 |
885 | with patch(
886 | "maverick_mcp.utils.orchestration_logging.get_orchestration_logger"
887 | ) as mock_get_logger:
888 | mock_logger = Mock()
889 | mock_get_logger.return_value = mock_logger
890 |
891 | # Get baseline memory
892 | gc.collect()
893 | initial_objects = len(gc.get_objects())
894 |
895 | # Perform many logging operations
896 | for i in range(50):
897 | logger = OrchestrationLogger(f"TestComponent_{i}")
898 | logger.set_request_context(
899 | session_id=f"session_{i}",
900 | request_id=f"req_{i}",
901 | large_data=f"data_{'x' * 100}_{i}", # Some larger context data
902 | )
903 | logger.info("Test message", param1=f"value_{i}", param2=i)
904 |
905 | # Check memory growth
906 | gc.collect()
907 | final_objects = len(gc.get_objects())
908 |
909 | # Memory growth should be reasonable (not growing indefinitely)
910 | object_growth = final_objects - initial_objects
911 | assert object_growth < 1000 # Reasonable threshold for test
912 |
```
--------------------------------------------------------------------------------
/scripts/load_tiingo_data.py:
--------------------------------------------------------------------------------
```python
1 | #!/usr/bin/env python3
2 | """
3 | Tiingo Data Loader for MaverickMCP
4 |
5 | Loads market data from Tiingo API into the self-contained MaverickMCP database.
6 | Supports batch loading, rate limiting, progress tracking, and technical indicator calculation.
7 | """
8 |
9 | import argparse
10 | import asyncio
11 | import json
12 | import logging
13 | import os
14 | import sys
15 | from datetime import datetime, timedelta
16 | from pathlib import Path
17 | from typing import Any
18 |
19 | import aiohttp
20 | import numpy as np
21 | import pandas as pd
22 | from sqlalchemy import create_engine
23 | from sqlalchemy.orm import sessionmaker
24 | from tqdm import tqdm
25 |
26 | # Add parent directory to path for imports
27 | sys.path.append(str(Path(__file__).parent.parent))
28 |
29 | from maverick_mcp.data.models import (
30 | Stock,
31 | bulk_insert_price_data,
32 | )
33 |
34 | # Configure logging
35 | logging.basicConfig(
36 | level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
37 | )
38 | logger = logging.getLogger(__name__)
39 |
40 | # Configuration - following tiingo-python patterns from api.py
41 | # Base URL without version suffix (will be added per endpoint)
42 | DEFAULT_BASE_URL = os.getenv("TIINGO_BASE_URL", "https://api.tiingo.com")
43 |
44 | # API token from environment
45 | TIINGO_API_TOKEN = os.getenv("TIINGO_API_TOKEN")
46 |
47 | # Rate limiting configuration - can be overridden by command line
48 | DEFAULT_MAX_CONCURRENT = int(os.getenv("TIINGO_MAX_CONCURRENT", "5"))
49 | DEFAULT_RATE_LIMIT_PER_HOUR = int(os.getenv("TIINGO_RATE_LIMIT", "2400"))
50 | DEFAULT_CHECKPOINT_FILE = os.getenv(
51 | "TIINGO_CHECKPOINT_FILE", "tiingo_load_progress.json"
52 | )
53 |
54 | # Default timeout for requests (from tiingo-python)
55 | DEFAULT_TIMEOUT = int(os.getenv("TIINGO_TIMEOUT", "10"))
56 |
57 |
58 | class TiingoDataLoader:
59 | """Handles loading data from Tiingo API into MaverickMCP database.
60 |
61 | Following the design patterns from tiingo-python library.
62 | """
63 |
64 | def __init__(
65 | self,
66 | api_token: str | None = None,
67 | db_url: str | None = None,
68 | base_url: str | None = None,
69 | timeout: int | None = None,
70 | rate_limit_per_hour: int | None = None,
71 | checkpoint_file: str | None = None,
72 | ):
73 | """Initialize the Tiingo data loader.
74 |
75 | Args:
76 | api_token: Tiingo API token (defaults to env var)
77 | db_url: Database URL (defaults to env var)
78 | base_url: Base URL for Tiingo API (defaults to env var)
79 | timeout: Request timeout in seconds
80 | rate_limit_per_hour: Max requests per hour
81 | checkpoint_file: Path to checkpoint file
82 | """
83 | # API configuration (following tiingo-python patterns)
84 | self.api_token = api_token or TIINGO_API_TOKEN
85 | if not self.api_token:
86 | raise ValueError(
87 | "API token required. Set TIINGO_API_TOKEN env var or pass api_token parameter."
88 | )
89 |
90 | # Database configuration
91 | self.db_url = db_url or os.getenv("DATABASE_URL")
92 | if not self.db_url:
93 | raise ValueError(
94 | "Database URL required. Set DATABASE_URL env var or pass db_url parameter."
95 | )
96 |
97 | self.engine = create_engine(self.db_url)
98 | self.SessionLocal = sessionmaker(bind=self.engine)
99 |
100 | # API endpoint configuration
101 | self.base_url = base_url or DEFAULT_BASE_URL
102 | self.timeout = timeout or DEFAULT_TIMEOUT
103 |
104 | # Rate limiting
105 | self.request_count = 0
106 | self.start_time = datetime.now()
107 | self.rate_limit_per_hour = rate_limit_per_hour or DEFAULT_RATE_LIMIT_PER_HOUR
108 | self.rate_limit_delay = (
109 | 3600 / self.rate_limit_per_hour
110 | ) # seconds between requests
111 |
112 | # Checkpoint configuration
113 | self.checkpoint_file = checkpoint_file or DEFAULT_CHECKPOINT_FILE
114 | self.checkpoint_data = self.load_checkpoint()
115 |
116 | # Session configuration (following tiingo-python)
117 | self._session = None
118 |
119 | def load_checkpoint(self) -> dict:
120 | """Load checkpoint data if exists."""
121 | if Path(self.checkpoint_file).exists():
122 | try:
123 | with open(self.checkpoint_file) as f:
124 | return json.load(f)
125 | except Exception as e:
126 | logger.warning(f"Could not load checkpoint: {e}")
127 | return {"completed_symbols": [], "last_symbol": None}
128 |
129 | def save_checkpoint(self, symbol: str):
130 | """Save checkpoint data."""
131 | self.checkpoint_data["completed_symbols"].append(symbol)
132 | self.checkpoint_data["last_symbol"] = symbol
133 | self.checkpoint_data["timestamp"] = datetime.now().isoformat()
134 |
135 | try:
136 | with open(self.checkpoint_file, "w") as f:
137 | json.dump(self.checkpoint_data, f, indent=2)
138 | except Exception as e:
139 | logger.error(f"Could not save checkpoint: {e}")
140 |
141 | def _get_headers(self) -> dict[str, str]:
142 | """Get request headers following tiingo-python patterns."""
143 | return {
144 | "Content-Type": "application/json",
145 | "Authorization": f"Token {self.api_token}",
146 | "User-Agent": "tiingo-python-client/maverick-mcp",
147 | }
148 |
149 | async def _request(
150 | self,
151 | session: aiohttp.ClientSession,
152 | endpoint: str,
153 | params: dict[str, Any] | None = None,
154 | max_retries: int = 3,
155 | ) -> Any | None:
156 | """Make HTTP request with rate limiting and error handling.
157 |
158 | Following tiingo-python's request patterns from api.py.
159 |
160 | Args:
161 | session: aiohttp session
162 | endpoint: API endpoint (will be appended to base_url)
163 | params: Query parameters
164 | max_retries: Maximum number of retries
165 |
166 | Returns:
167 | Response data or None if failed
168 | """
169 | # Rate limiting
170 | await asyncio.sleep(self.rate_limit_delay)
171 | self.request_count += 1
172 |
173 | # Build URL
174 | url = f"{self.base_url}{endpoint}"
175 | if params:
176 | param_str = "&".join([f"{k}={v}" for k, v in params.items()])
177 | url = f"{url}?{param_str}"
178 |
179 | headers = self._get_headers()
180 |
181 | for attempt in range(max_retries):
182 | try:
183 | timeout = aiohttp.ClientTimeout(total=self.timeout)
184 | async with session.get(
185 | url, headers=headers, timeout=timeout
186 | ) as response:
187 | if response.status == 200:
188 | return await response.json()
189 | elif response.status == 400:
190 | error_msg = await response.text()
191 | logger.error(f"Bad request (400): {error_msg}")
192 | return None
193 | elif response.status == 404:
194 | logger.warning(f"Resource not found (404): {endpoint}")
195 | return None
196 | elif response.status == 429:
197 | # Rate limited, exponential backoff
198 | retry_after = response.headers.get("Retry-After")
199 | if retry_after:
200 | wait_time = int(retry_after)
201 | else:
202 | wait_time = min(60 * (2**attempt), 300)
203 | logger.warning(f"Rate limited, waiting {wait_time}s...")
204 | await asyncio.sleep(wait_time)
205 | continue
206 | elif response.status >= 500:
207 | # Server error, retry with backoff
208 | if attempt < max_retries - 1:
209 | wait_time = min(5 * (2**attempt), 60)
210 | logger.warning(
211 | f"Server error {response.status}, retry in {wait_time}s..."
212 | )
213 | await asyncio.sleep(wait_time)
214 | continue
215 | else:
216 | logger.error(f"Server error after {max_retries} attempts")
217 | return None
218 | else:
219 | error_text = await response.text()
220 | logger.error(f"HTTP {response.status}: {error_text}")
221 | return None
222 |
223 | except TimeoutError:
224 | if attempt < max_retries - 1:
225 | wait_time = min(10 * (2**attempt), 60)
226 | logger.warning(f"Timeout, retry in {wait_time}s...")
227 | await asyncio.sleep(wait_time)
228 | continue
229 | else:
230 | logger.error(f"Timeout after {max_retries} attempts")
231 | return None
232 |
233 | except Exception as e:
234 | if attempt < max_retries - 1:
235 | wait_time = min(10 * (2**attempt), 60)
236 | logger.warning(f"Error: {e}, retry in {wait_time}s...")
237 | await asyncio.sleep(wait_time)
238 | continue
239 | else:
240 | logger.error(f"Failed after {max_retries} attempts: {e}")
241 | return None
242 |
243 | return None
244 |
245 | def _process_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
246 | """Process raw DataFrame from Tiingo API.
247 |
248 | Following tiingo-python's DataFrame processing patterns.
249 |
250 | Args:
251 | df: Raw DataFrame from API
252 |
253 | Returns:
254 | Processed DataFrame with proper column names and index
255 | """
256 | if df.empty:
257 | return df
258 |
259 | # Handle date column following tiingo-python
260 | if "date" in df.columns:
261 | df["date"] = pd.to_datetime(df["date"])
262 |
263 | # Standardize column names to match expected format
264 | column_mapping = {
265 | "open": "Open",
266 | "high": "High",
267 | "low": "Low",
268 | "close": "Close",
269 | "volume": "Volume",
270 | "adjOpen": "Adj Open",
271 | "adjHigh": "Adj High",
272 | "adjLow": "Adj Low",
273 | "adjClose": "Adj Close",
274 | "adjVolume": "Adj Volume",
275 | "divCash": "Dividend",
276 | "splitFactor": "Split Factor",
277 | }
278 |
279 | # Only rename columns that exist
280 | rename_dict = {
281 | old: new for old, new in column_mapping.items() if old in df.columns
282 | }
283 | if rename_dict:
284 | df = df.rename(columns=rename_dict)
285 |
286 | # Set date as index
287 | df = df.set_index("date")
288 |
289 | # Localize to UTC following tiingo-python approach
290 | if df.index.tz is None:
291 | df.index = df.index.tz_localize("UTC")
292 |
293 | # For database storage, convert to date only (no time component)
294 | df.index = df.index.date
295 |
296 | return df
297 |
298 | async def get_ticker_metadata(
299 | self, session: aiohttp.ClientSession, symbol: str
300 | ) -> dict[str, Any] | None:
301 | """Get metadata for a specific ticker.
302 |
303 | Following tiingo-python's get_ticker_metadata pattern.
304 | """
305 | endpoint = f"/tiingo/daily/{symbol}"
306 | return await self._request(session, endpoint)
307 |
308 | async def get_available_symbols(
309 | self,
310 | session: aiohttp.ClientSession,
311 | asset_types: list[str] | None = None,
312 | exchanges: list[str] | None = None,
313 | ) -> list[str]:
314 | """Get list of available symbols from Tiingo with optional filtering.
315 |
316 | Following tiingo-python's list_tickers pattern.
317 | """
318 | endpoint = "/tiingo/daily/supported_tickers"
319 | data = await self._request(session, endpoint)
320 |
321 | if data:
322 | # Default filters if not provided
323 | asset_types = asset_types or ["Stock"]
324 | exchanges = exchanges or ["NYSE", "NASDAQ"]
325 |
326 | symbols = []
327 | for ticker_info in data:
328 | if (
329 | ticker_info.get("exchange") in exchanges
330 | and ticker_info.get("assetType") in asset_types
331 | and ticker_info.get("priceCurrency") == "USD"
332 | ):
333 | symbols.append(ticker_info["ticker"])
334 | return symbols
335 | return []
336 |
337 | async def get_daily_price_history(
338 | self,
339 | session: aiohttp.ClientSession,
340 | symbol: str,
341 | start_date: str | None = None,
342 | end_date: str | None = None,
343 | frequency: str = "daily",
344 | columns: list[str] | None = None,
345 | ) -> pd.DataFrame:
346 | """Fetch historical price data for a symbol.
347 |
348 | Following tiingo-python's get_dataframe pattern from api.py.
349 |
350 | Args:
351 | session: aiohttp session
352 | symbol: Stock ticker symbol
353 | start_date: Start date in YYYY-MM-DD format
354 | end_date: End date in YYYY-MM-DD format
355 | frequency: Data frequency (daily, weekly, monthly, annually)
356 | columns: Specific columns to return
357 |
358 | Returns:
359 | DataFrame with price history
360 | """
361 | endpoint = f"/tiingo/daily/{symbol}/prices"
362 |
363 | # Build params following tiingo-python
364 | params = {
365 | "format": "json",
366 | "resampleFreq": frequency,
367 | }
368 |
369 | if start_date:
370 | params["startDate"] = start_date
371 | if end_date:
372 | params["endDate"] = end_date
373 | if columns:
374 | params["columns"] = ",".join(columns)
375 |
376 | data = await self._request(session, endpoint, params)
377 |
378 | if data:
379 | try:
380 | df = pd.DataFrame(data)
381 | if not df.empty:
382 | # Process DataFrame following tiingo-python patterns
383 | df = self._process_dataframe(df)
384 |
385 | # Validate data integrity
386 | if len(df) == 0:
387 | logger.warning(f"Empty dataset returned for {symbol}")
388 | return pd.DataFrame()
389 |
390 | # Check for required columns
391 | required_cols = ["Open", "High", "Low", "Close", "Volume"]
392 | missing_cols = [
393 | col for col in required_cols if col not in df.columns
394 | ]
395 | if missing_cols:
396 | logger.warning(f"Missing columns for {symbol}: {missing_cols}")
397 |
398 | return df
399 |
400 | except Exception as e:
401 | logger.error(f"Error processing data for {symbol}: {e}")
402 | return pd.DataFrame()
403 |
404 | return pd.DataFrame()
405 |
406 | def calculate_technical_indicators(self, df: pd.DataFrame) -> pd.DataFrame:
407 | """Calculate technical indicators for the data."""
408 | if df.empty or len(df) < 200:
409 | return df
410 |
411 | try:
412 | # Moving averages
413 | df["SMA_20"] = df["Close"].rolling(window=20).mean()
414 | df["SMA_50"] = df["Close"].rolling(window=50).mean()
415 | df["SMA_150"] = df["Close"].rolling(window=150).mean()
416 | df["SMA_200"] = df["Close"].rolling(window=200).mean()
417 | df["EMA_21"] = df["Close"].ewm(span=21, adjust=False).mean()
418 |
419 | # RSI
420 | delta = df["Close"].diff()
421 | gain = (delta.where(delta > 0, 0)).rolling(window=14).mean()
422 | loss = (-delta.where(delta < 0, 0)).rolling(window=14).mean()
423 | rs = gain / loss
424 | df["RSI"] = 100 - (100 / (1 + rs))
425 |
426 | # MACD
427 | exp1 = df["Close"].ewm(span=12, adjust=False).mean()
428 | exp2 = df["Close"].ewm(span=26, adjust=False).mean()
429 | df["MACD"] = exp1 - exp2
430 | df["MACD_Signal"] = df["MACD"].ewm(span=9, adjust=False).mean()
431 | df["MACD_Histogram"] = df["MACD"] - df["MACD_Signal"]
432 |
433 | # ATR
434 | high_low = df["High"] - df["Low"]
435 | high_close = np.abs(df["High"] - df["Close"].shift())
436 | low_close = np.abs(df["Low"] - df["Close"].shift())
437 | ranges = pd.concat([high_low, high_close, low_close], axis=1)
438 | true_range = np.max(ranges, axis=1)
439 | df["ATR"] = true_range.rolling(14).mean()
440 |
441 | # ADR (Average Daily Range) as percentage
442 | df["ADR_PCT"] = (
443 | ((df["High"] - df["Low"]) / df["Close"] * 100).rolling(20).mean()
444 | )
445 |
446 | # Volume indicators
447 | df["Volume_SMA_30"] = df["Volume"].rolling(window=30).mean()
448 | df["Volume_Ratio"] = df["Volume"] / df["Volume_SMA_30"]
449 |
450 | # Momentum Score (simplified)
451 | returns = df["Close"].pct_change(periods=252) # 1-year returns
452 | df["Momentum_Score"] = returns.rank(pct=True) * 100
453 |
454 | except Exception as e:
455 | logger.error(f"Error calculating indicators: {e}")
456 |
457 | return df
458 |
459 | def run_maverick_screening(self, df: pd.DataFrame, symbol: str) -> dict | None:
460 | """Run Maverick momentum screening algorithm."""
461 | if df.empty or len(df) < 200:
462 | return None
463 |
464 | try:
465 | latest = df.iloc[-1]
466 |
467 | # Maverick criteria
468 | price_above_ema21 = latest["Close"] > latest.get("EMA_21", 0)
469 | ema21_above_sma50 = latest.get("EMA_21", 0) > latest.get("SMA_50", 0)
470 | sma50_above_sma200 = latest.get("SMA_50", 0) > latest.get("SMA_200", 0)
471 | strong_momentum = latest.get("Momentum_Score", 0) > 70
472 |
473 | # Calculate combined score
474 | score = 0
475 | if price_above_ema21:
476 | score += 25
477 | if ema21_above_sma50:
478 | score += 25
479 | if sma50_above_sma200:
480 | score += 25
481 | if strong_momentum:
482 | score += 25
483 |
484 | if score >= 75: # Meets criteria
485 | return {
486 | "stock": symbol,
487 | "close": float(latest["Close"]),
488 | "volume": int(latest["Volume"]),
489 | "momentum_score": float(latest.get("Momentum_Score", 0)),
490 | "combined_score": score,
491 | "adr_pct": float(latest.get("ADR_PCT", 0)),
492 | "atr": float(latest.get("ATR", 0)),
493 | "ema_21": float(latest.get("EMA_21", 0)),
494 | "sma_50": float(latest.get("SMA_50", 0)),
495 | "sma_150": float(latest.get("SMA_150", 0)),
496 | "sma_200": float(latest.get("SMA_200", 0)),
497 | }
498 | except Exception as e:
499 | logger.error(f"Error in Maverick screening for {symbol}: {e}")
500 |
501 | return None
502 |
503 | def run_bear_screening(self, df: pd.DataFrame, symbol: str) -> dict | None:
504 | """Run Bear market screening algorithm."""
505 | if df.empty or len(df) < 200:
506 | return None
507 |
508 | try:
509 | latest = df.iloc[-1]
510 |
511 | # Bear criteria
512 | price_below_ema21 = latest["Close"] < latest.get("EMA_21", float("inf"))
513 | ema21_below_sma50 = latest.get("EMA_21", float("inf")) < latest.get(
514 | "SMA_50", float("inf")
515 | )
516 | weak_momentum = latest.get("Momentum_Score", 100) < 30
517 | negative_macd = latest.get("MACD", 0) < 0
518 |
519 | # Calculate bear score
520 | score = 0
521 | if price_below_ema21:
522 | score += 25
523 | if ema21_below_sma50:
524 | score += 25
525 | if weak_momentum:
526 | score += 25
527 | if negative_macd:
528 | score += 25
529 |
530 | if score >= 75: # Meets bear criteria
531 | return {
532 | "stock": symbol,
533 | "close": float(latest["Close"]),
534 | "volume": int(latest["Volume"]),
535 | "momentum_score": float(latest.get("Momentum_Score", 0)),
536 | "score": score,
537 | "rsi_14": float(latest.get("RSI", 0)),
538 | "macd": float(latest.get("MACD", 0)),
539 | "macd_signal": float(latest.get("MACD_Signal", 0)),
540 | "macd_histogram": float(latest.get("MACD_Histogram", 0)),
541 | "adr_pct": float(latest.get("ADR_PCT", 0)),
542 | "atr": float(latest.get("ATR", 0)),
543 | "ema_21": float(latest.get("EMA_21", 0)),
544 | "sma_50": float(latest.get("SMA_50", 0)),
545 | "sma_200": float(latest.get("SMA_200", 0)),
546 | }
547 | except Exception as e:
548 | logger.error(f"Error in Bear screening for {symbol}: {e}")
549 |
550 | return None
551 |
552 | def run_supply_demand_screening(self, df: pd.DataFrame, symbol: str) -> dict | None:
553 | """Run Supply/Demand breakout screening algorithm."""
554 | if df.empty or len(df) < 200:
555 | return None
556 |
557 | try:
558 | latest = df.iloc[-1]
559 |
560 | # Supply/Demand criteria (accumulation phase)
561 | close = latest["Close"]
562 | sma_50 = latest.get("SMA_50", 0)
563 | sma_150 = latest.get("SMA_150", 0)
564 | sma_200 = latest.get("SMA_200", 0)
565 |
566 | # Check for proper alignment
567 | price_above_all = close > sma_50 > sma_150 > sma_200
568 | strong_momentum = latest.get("Momentum_Score", 0) > 80
569 |
570 | # Volume confirmation
571 | volume_confirmation = latest.get("Volume_Ratio", 0) > 1.2
572 |
573 | if price_above_all and strong_momentum and volume_confirmation:
574 | return {
575 | "stock": symbol,
576 | "close": float(close),
577 | "volume": int(latest["Volume"]),
578 | "momentum_score": float(latest.get("Momentum_Score", 0)),
579 | "adr_pct": float(latest.get("ADR_PCT", 0)),
580 | "atr": float(latest.get("ATR", 0)),
581 | "ema_21": float(latest.get("EMA_21", 0)),
582 | "sma_50": float(sma_50),
583 | "sma_150": float(sma_150),
584 | "sma_200": float(sma_200),
585 | "avg_volume_30d": float(latest.get("Volume_SMA_30", 0)),
586 | }
587 | except Exception as e:
588 | logger.error(f"Error in Supply/Demand screening for {symbol}: {e}")
589 |
590 | return None
591 |
592 | async def process_symbol(
593 | self,
594 | session: aiohttp.ClientSession,
595 | symbol: str,
596 | start_date: str,
597 | end_date: str,
598 | calculate_indicators: bool = True,
599 | run_screening: bool = True,
600 | ) -> tuple[bool, dict | None]:
601 | """Process a single symbol - fetch data, calculate indicators, run screening."""
602 | try:
603 | # Skip if already processed
604 | if symbol in self.checkpoint_data.get("completed_symbols", []):
605 | logger.info(f"Skipping {symbol} - already processed")
606 | return True, None
607 |
608 | # Fetch historical data using tiingo-python pattern
609 | df = await self.get_daily_price_history(
610 | session, symbol, start_date, end_date
611 | )
612 |
613 | if df.empty:
614 | logger.warning(f"No data available for {symbol}")
615 | return False, None
616 |
617 | # Store in database
618 | with self.SessionLocal() as db_session:
619 | # Create or get stock record
620 | Stock.get_or_create(db_session, symbol)
621 |
622 | # Bulk insert price data
623 | records_inserted = bulk_insert_price_data(db_session, symbol, df)
624 | logger.info(f"Inserted {records_inserted} records for {symbol}")
625 |
626 | screening_results = {}
627 |
628 | if calculate_indicators:
629 | # Calculate technical indicators
630 | df = self.calculate_technical_indicators(df)
631 |
632 | if run_screening:
633 | # Run screening algorithms
634 | maverick_result = self.run_maverick_screening(df, symbol)
635 | if maverick_result:
636 | screening_results["maverick"] = maverick_result
637 |
638 | bear_result = self.run_bear_screening(df, symbol)
639 | if bear_result:
640 | screening_results["bear"] = bear_result
641 |
642 | supply_demand_result = self.run_supply_demand_screening(df, symbol)
643 | if supply_demand_result:
644 | screening_results["supply_demand"] = supply_demand_result
645 |
646 | # Save checkpoint
647 | self.save_checkpoint(symbol)
648 |
649 | return True, screening_results
650 |
651 | except Exception as e:
652 | logger.error(f"Error processing {symbol}: {e}")
653 | return False, None
654 |
655 | async def load_symbols(
656 | self,
657 | symbols: list[str],
658 | start_date: str,
659 | end_date: str,
660 | calculate_indicators: bool = True,
661 | run_screening: bool = True,
662 | max_concurrent: int = None,
663 | ):
664 | """Load data for multiple symbols with concurrent processing."""
665 | logger.info(
666 | f"Loading data for {len(symbols)} symbols from {start_date} to {end_date}"
667 | )
668 |
669 | # Filter out already processed symbols if resuming
670 | symbols_to_process = [
671 | s
672 | for s in symbols
673 | if s not in self.checkpoint_data.get("completed_symbols", [])
674 | ]
675 |
676 | if len(symbols_to_process) < len(symbols):
677 | logger.info(
678 | f"Resuming: {len(symbols) - len(symbols_to_process)} symbols already processed"
679 | )
680 |
681 | screening_results = {"maverick": [], "bear": [], "supply_demand": []}
682 |
683 | # Use provided max_concurrent or default
684 | concurrent_limit = max_concurrent or DEFAULT_MAX_CONCURRENT
685 |
686 | async with aiohttp.ClientSession() as session:
687 | # Process in batches with semaphore for rate limiting
688 | semaphore = asyncio.Semaphore(concurrent_limit)
689 |
690 | async def process_with_semaphore(symbol):
691 | async with semaphore:
692 | return await self.process_symbol(
693 | session,
694 | symbol,
695 | start_date,
696 | end_date,
697 | calculate_indicators,
698 | run_screening,
699 | )
700 |
701 | # Create tasks with progress bar
702 | tasks = []
703 | for symbol in symbols_to_process:
704 | tasks.append(process_with_semaphore(symbol))
705 |
706 | # Process with progress bar
707 | with tqdm(total=len(tasks), desc="Processing symbols") as pbar:
708 | for coro in asyncio.as_completed(tasks):
709 | success, results = await coro
710 | if results:
711 | for screen_type, data in results.items():
712 | screening_results[screen_type].append(data)
713 | pbar.update(1)
714 |
715 | # Store screening results in database
716 | if run_screening:
717 | self.store_screening_results(screening_results)
718 |
719 | logger.info(f"Completed loading {len(symbols_to_process)} symbols")
720 | logger.info(
721 | f"Screening results - Maverick: {len(screening_results['maverick'])}, "
722 | f"Bear: {len(screening_results['bear'])}, "
723 | f"Supply/Demand: {len(screening_results['supply_demand'])}"
724 | )
725 |
726 | def store_screening_results(self, results: dict):
727 | """Store screening results in database."""
728 | with self.SessionLocal() as db_session:
729 | # Store Maverick results
730 | for _data in results["maverick"]:
731 | # Implementation would create MaverickStocks records
732 | pass
733 |
734 | # Store Bear results
735 | for _data in results["bear"]:
736 | # Implementation would create MaverickBearStocks records
737 | pass
738 |
739 | # Store Supply/Demand results
740 | for _data in results["supply_demand"]:
741 | # Implementation would create SupplyDemandBreakoutStocks records
742 | pass
743 |
744 | db_session.commit()
745 |
746 |
747 | def get_test_symbols() -> list[str]:
748 | """Get a small test set of symbols for development.
749 |
750 | These are just for testing - production use should load from
751 | external sources or command line arguments.
752 | """
753 | # Test symbols from different sectors for comprehensive testing
754 | return [
755 | "AAPL", # Apple - Tech
756 | "MSFT", # Microsoft - Tech
757 | "GOOGL", # Alphabet - Tech
758 | "AMZN", # Amazon - Consumer Discretionary
759 | "NVDA", # NVIDIA - Tech
760 | "META", # Meta - Communication
761 | "TSLA", # Tesla - Consumer Discretionary
762 | "UNH", # UnitedHealth - Healthcare
763 | "JPM", # JPMorgan Chase - Financials
764 | "V", # Visa - Financials
765 | "WMT", # Walmart - Consumer Staples
766 | "JNJ", # Johnson & Johnson - Healthcare
767 | "MA", # Mastercard - Financials
768 | "HD", # Home Depot - Consumer Discretionary
769 | "PG", # Procter & Gamble - Consumer Staples
770 | "XOM", # ExxonMobil - Energy
771 | "CVX", # Chevron - Energy
772 | "KO", # Coca-Cola - Consumer Staples
773 | "PEP", # PepsiCo - Consumer Staples
774 | "ADBE", # Adobe - Tech
775 | "NFLX", # Netflix - Communication
776 | "CRM", # Salesforce - Tech
777 | "DIS", # Disney - Communication
778 | "COST", # Costco - Consumer Staples
779 | "MRK", # Merck - Healthcare
780 | ]
781 |
782 |
783 | def get_sp500_symbols() -> list[str]:
784 | """Get S&P 500 symbols list from external source or file.
785 |
786 | This function should load S&P 500 symbols from:
787 | 1. Environment variable SP500_SYMBOLS_FILE pointing to a file
788 | 2. Download from a public data source
789 | 3. Return empty list with warning if unavailable
790 | """
791 | # Try to load from file specified in environment
792 | symbols_file = os.getenv("SP500_SYMBOLS_FILE")
793 | if symbols_file and Path(symbols_file).exists():
794 | try:
795 | with open(symbols_file) as f:
796 | symbols = [line.strip() for line in f if line.strip()]
797 | logger.info(
798 | f"Loaded {len(symbols)} S&P 500 symbols from {symbols_file}"
799 | )
800 | return symbols
801 | except Exception as e:
802 | logger.warning(f"Could not load S&P 500 symbols from {symbols_file}: {e}")
803 |
804 | # Try to fetch from a public source (like Wikipedia or Yahoo Finance)
805 | try:
806 | # Using pandas to read S&P 500 list from Wikipedia
807 | url = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
808 | tables = pd.read_html(url)
809 | sp500_table = tables[0] # First table contains the S&P 500 list
810 | symbols = sp500_table["Symbol"].tolist()
811 | logger.info(f"Fetched {len(symbols)} S&P 500 symbols from Wikipedia")
812 |
813 | # Optional: Save to cache file for future use
814 | cache_file = os.getenv("SP500_CACHE_FILE", "sp500_symbols_cache.txt")
815 | try:
816 | with open(cache_file, "w") as f:
817 | for symbol in symbols:
818 | f.write(f"{symbol}\n")
819 | logger.info(f"Cached S&P 500 symbols to {cache_file}")
820 | except Exception as e:
821 | logger.debug(f"Could not cache symbols: {e}")
822 |
823 | return symbols
824 |
825 | except Exception as e:
826 | logger.warning(f"Could not fetch S&P 500 symbols from web: {e}")
827 |
828 | # Try to load from cache if web fetch failed
829 | cache_file = os.getenv("SP500_CACHE_FILE", "sp500_symbols_cache.txt")
830 | if Path(cache_file).exists():
831 | try:
832 | with open(cache_file) as f:
833 | symbols = [line.strip() for line in f if line.strip()]
834 | logger.info(f"Loaded {len(symbols)} S&P 500 symbols from cache")
835 | return symbols
836 | except Exception as e:
837 | logger.warning(f"Could not load from cache: {e}")
838 |
839 | logger.error("Unable to load S&P 500 symbols. Please specify --file or --symbols")
840 | return []
841 |
842 |
843 | def main():
844 | """Main entry point."""
845 | parser = argparse.ArgumentParser(description="Load market data from Tiingo API")
846 | parser.add_argument("--symbols", nargs="+", help="List of symbols to load")
847 | parser.add_argument("--file", help="Load symbols from file (one per line)")
848 | parser.add_argument(
849 | "--test", action="store_true", help="Load test set of 25 symbols"
850 | )
851 | parser.add_argument("--sp500", action="store_true", help="Load S&P 500 symbols")
852 | parser.add_argument(
853 | "--years", type=int, default=2, help="Number of years of history"
854 | )
855 | parser.add_argument("--start-date", help="Start date (YYYY-MM-DD)")
856 | parser.add_argument("--end-date", help="End date (YYYY-MM-DD)")
857 | parser.add_argument(
858 | "--calculate-indicators",
859 | action="store_true",
860 | help="Calculate technical indicators",
861 | )
862 | parser.add_argument(
863 | "--run-screening", action="store_true", help="Run screening algorithms"
864 | )
865 | parser.add_argument(
866 | "--max-concurrent", type=int, default=5, help="Maximum concurrent requests"
867 | )
868 | parser.add_argument("--resume", action="store_true", help="Resume from checkpoint")
869 | parser.add_argument("--db-url", help="Database URL override")
870 |
871 | args = parser.parse_args()
872 |
873 | # Check for API token
874 | if not TIINGO_API_TOKEN:
875 | logger.error("TIINGO_API_TOKEN environment variable not set")
876 | sys.exit(1)
877 |
878 | # Determine database URL
879 | db_url = args.db_url or os.getenv("MCP_DATABASE_URL") or os.getenv("DATABASE_URL")
880 | if not db_url:
881 | logger.error("Database URL not configured")
882 | sys.exit(1)
883 |
884 | # Determine symbols to load
885 | symbols = []
886 | if args.symbols:
887 | symbols = args.symbols
888 | elif args.file:
889 | # Load symbols from file
890 | try:
891 | with open(args.file) as f:
892 | symbols = [line.strip() for line in f if line.strip()]
893 | logger.info(f"Loaded {len(symbols)} symbols from {args.file}")
894 | except Exception as e:
895 | logger.error(f"Could not read symbols from file: {e}")
896 | sys.exit(1)
897 | elif args.test:
898 | symbols = get_test_symbols()
899 | logger.info(f"Using test set of {len(symbols)} symbols")
900 | elif args.sp500:
901 | symbols = get_sp500_symbols()
902 | logger.info(f"Using S&P 500 symbols ({len(symbols)} total)")
903 | else:
904 | logger.error("No symbols specified. Use --symbols, --file, --test, or --sp500")
905 | sys.exit(1)
906 |
907 | # Determine date range
908 | end_date = args.end_date or datetime.now().strftime("%Y-%m-%d")
909 | if args.start_date:
910 | start_date = args.start_date
911 | else:
912 | start_date = (datetime.now() - timedelta(days=365 * args.years)).strftime(
913 | "%Y-%m-%d"
914 | )
915 |
916 | # Create loader using tiingo-python style initialization
917 | loader = TiingoDataLoader(
918 | api_token=TIINGO_API_TOKEN,
919 | db_url=db_url,
920 | rate_limit_per_hour=DEFAULT_RATE_LIMIT_PER_HOUR,
921 | )
922 |
923 | # Run async loading
924 | asyncio.run(
925 | loader.load_symbols(
926 | symbols,
927 | start_date,
928 | end_date,
929 | calculate_indicators=args.calculate_indicators,
930 | run_screening=args.run_screening,
931 | max_concurrent=args.max_concurrent,
932 | )
933 | )
934 |
935 | logger.info("Data loading complete!")
936 |
937 | # Clean up checkpoint if completed successfully
938 | checkpoint_file = DEFAULT_CHECKPOINT_FILE
939 | if not args.resume and Path(checkpoint_file).exists():
940 | os.remove(checkpoint_file)
941 | logger.info("Removed checkpoint file")
942 |
943 |
944 | if __name__ == "__main__":
945 | main()
946 |
```
--------------------------------------------------------------------------------
/tests/performance/test_stress.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Stress Testing for Resource Usage Under Load.
3 |
4 | This test suite covers:
5 | - Sustained load testing (1+ hour)
6 | - Memory leak detection over time
7 | - CPU utilization monitoring under stress
8 | - Database connection pool exhaustion
9 | - File descriptor limits testing
10 | - Network connection limits
11 | - Queue overflow scenarios
12 | - System stability under extreme conditions
13 | """
14 |
15 | import asyncio
16 | import gc
17 | import logging
18 | import resource
19 | import threading
20 | import time
21 | from dataclasses import dataclass
22 | from typing import Any
23 | from unittest.mock import Mock
24 |
25 | import numpy as np
26 | import pandas as pd
27 | import psutil
28 | import pytest
29 |
30 | from maverick_mcp.backtesting import VectorBTEngine
31 | from maverick_mcp.backtesting.persistence import BacktestPersistenceManager
32 | from maverick_mcp.backtesting.strategies import STRATEGY_TEMPLATES
33 |
34 | logger = logging.getLogger(__name__)
35 |
36 |
37 | @dataclass
38 | class ResourceSnapshot:
39 | """Snapshot of system resources at a point in time."""
40 |
41 | timestamp: float
42 | memory_rss_mb: float
43 | memory_vms_mb: float
44 | memory_percent: float
45 | cpu_percent: float
46 | threads: int
47 | file_descriptors: int
48 | connections: int
49 | swap_usage_mb: float
50 |
51 |
52 | class ResourceMonitor:
53 | """Monitor system resources over time."""
54 |
55 | def __init__(self, interval: float = 1.0):
56 | self.interval = interval
57 | self.snapshots: list[ResourceSnapshot] = []
58 | self.monitoring = False
59 | self.monitor_thread = None
60 | self.process = psutil.Process()
61 |
62 | def start_monitoring(self):
63 | """Start continuous resource monitoring."""
64 | self.monitoring = True
65 | self.monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True)
66 | self.monitor_thread.start()
67 | logger.info("Resource monitoring started")
68 |
69 | def stop_monitoring(self):
70 | """Stop resource monitoring."""
71 | self.monitoring = False
72 | if self.monitor_thread:
73 | self.monitor_thread.join(timeout=2.0)
74 | logger.info(
75 | f"Resource monitoring stopped. Collected {len(self.snapshots)} snapshots"
76 | )
77 |
78 | def _monitor_loop(self):
79 | """Continuous monitoring loop."""
80 | while self.monitoring:
81 | try:
82 | snapshot = self._take_snapshot()
83 | self.snapshots.append(snapshot)
84 | time.sleep(self.interval)
85 | except Exception as e:
86 | logger.error(f"Error in resource monitoring: {e}")
87 |
88 | def _take_snapshot(self) -> ResourceSnapshot:
89 | """Take a resource snapshot."""
90 | memory_info = self.process.memory_info()
91 |
92 | # Get file descriptor count
93 | try:
94 | fd_count = self.process.num_fds()
95 | except AttributeError:
96 | # Windows doesn't have num_fds()
97 | fd_count = len(self.process.open_files())
98 |
99 | # Get connection count
100 | try:
101 | connections = len(self.process.connections())
102 | except (psutil.AccessDenied, psutil.NoSuchProcess):
103 | connections = 0
104 |
105 | # Get swap usage
106 | try:
107 | swap = psutil.swap_memory()
108 | swap_used_mb = swap.used / 1024 / 1024
109 | except Exception:
110 | swap_used_mb = 0
111 |
112 | return ResourceSnapshot(
113 | timestamp=time.time(),
114 | memory_rss_mb=memory_info.rss / 1024 / 1024,
115 | memory_vms_mb=memory_info.vms / 1024 / 1024,
116 | memory_percent=self.process.memory_percent(),
117 | cpu_percent=self.process.cpu_percent(),
118 | threads=self.process.num_threads(),
119 | file_descriptors=fd_count,
120 | connections=connections,
121 | swap_usage_mb=swap_used_mb,
122 | )
123 |
124 | def get_current_snapshot(self) -> ResourceSnapshot:
125 | """Get current resource snapshot."""
126 | return self._take_snapshot()
127 |
128 | def analyze_trends(self) -> dict[str, Any]:
129 | """Analyze resource usage trends."""
130 | if len(self.snapshots) < 2:
131 | return {"error": "Insufficient data for trend analysis"}
132 |
133 | # Calculate trends
134 | timestamps = [s.timestamp for s in self.snapshots]
135 | memories = [s.memory_rss_mb for s in self.snapshots]
136 | cpus = [s.cpu_percent for s in self.snapshots]
137 | fds = [s.file_descriptors for s in self.snapshots]
138 |
139 | # Linear regression for memory trend
140 | n = len(timestamps)
141 | sum_t = sum(timestamps)
142 | sum_m = sum(memories)
143 | sum_tm = sum(t * m for t, m in zip(timestamps, memories, strict=False))
144 | sum_tt = sum(t * t for t in timestamps)
145 |
146 | memory_slope = (
147 | (n * sum_tm - sum_t * sum_m) / (n * sum_tt - sum_t * sum_t)
148 | if n * sum_tt != sum_t * sum_t
149 | else 0
150 | )
151 |
152 | return {
153 | "duration_seconds": timestamps[-1] - timestamps[0],
154 | "initial_memory_mb": memories[0],
155 | "final_memory_mb": memories[-1],
156 | "memory_growth_mb": memories[-1] - memories[0],
157 | "memory_growth_rate_mb_per_hour": memory_slope * 3600,
158 | "peak_memory_mb": max(memories),
159 | "avg_cpu_percent": sum(cpus) / len(cpus),
160 | "peak_cpu_percent": max(cpus),
161 | "initial_file_descriptors": fds[0],
162 | "final_file_descriptors": fds[-1],
163 | "fd_growth": fds[-1] - fds[0],
164 | "peak_file_descriptors": max(fds),
165 | "snapshots_count": len(self.snapshots),
166 | }
167 |
168 |
169 | class StressTestRunner:
170 | """Run various stress tests."""
171 |
172 | def __init__(self, data_provider):
173 | self.data_provider = data_provider
174 | self.resource_monitor = ResourceMonitor(interval=2.0)
175 |
176 | async def sustained_load_test(
177 | self, duration_minutes: int = 60, concurrent_load: int = 10
178 | ) -> dict[str, Any]:
179 | """Run sustained load test for extended duration."""
180 | logger.info(
181 | f"Starting sustained load test: {duration_minutes} minutes with {concurrent_load} concurrent operations"
182 | )
183 |
184 | self.resource_monitor.start_monitoring()
185 | start_time = time.time()
186 | end_time = start_time + (duration_minutes * 60)
187 |
188 | total_operations = 0
189 | total_errors = 0
190 | operation_times = []
191 |
192 | try:
193 | # Create semaphore for concurrent control
194 | semaphore = asyncio.Semaphore(concurrent_load)
195 |
196 | async def sustained_operation(operation_id: int):
197 | """Single sustained operation."""
198 | nonlocal total_operations, total_errors
199 |
200 | engine = VectorBTEngine(data_provider=self.data_provider)
201 | symbol = f"STRESS_{operation_id % 20}" # Cycle through 20 symbols
202 | strategy = ["sma_cross", "rsi", "macd"][
203 | operation_id % 3
204 | ] # Cycle through strategies
205 |
206 | try:
207 | async with semaphore:
208 | op_start = time.time()
209 |
210 | await engine.run_backtest(
211 | symbol=symbol,
212 | strategy_type=strategy,
213 | parameters=STRATEGY_TEMPLATES[strategy]["parameters"],
214 | start_date="2023-01-01",
215 | end_date="2023-12-31",
216 | )
217 |
218 | op_time = time.time() - op_start
219 | operation_times.append(op_time)
220 | total_operations += 1
221 |
222 | if total_operations % 100 == 0:
223 | logger.info(f"Completed {total_operations} operations")
224 |
225 | except Exception as e:
226 | total_errors += 1
227 | logger.error(f"Operation {operation_id} failed: {e}")
228 |
229 | # Run operations continuously until duration expires
230 | operation_id = 0
231 | active_tasks = []
232 |
233 | while time.time() < end_time:
234 | # Start new operation
235 | task = asyncio.create_task(sustained_operation(operation_id))
236 | active_tasks.append(task)
237 | operation_id += 1
238 |
239 | # Clean up completed tasks
240 | active_tasks = [t for t in active_tasks if not t.done()]
241 |
242 | # Control task creation rate
243 | await asyncio.sleep(0.1)
244 |
245 | # Prevent task accumulation
246 | if len(active_tasks) > concurrent_load * 2:
247 | await asyncio.sleep(1.0)
248 |
249 | # Wait for remaining tasks to complete
250 | if active_tasks:
251 | await asyncio.gather(*active_tasks, return_exceptions=True)
252 |
253 | finally:
254 | self.resource_monitor.stop_monitoring()
255 |
256 | actual_duration = time.time() - start_time
257 | trend_analysis = self.resource_monitor.analyze_trends()
258 |
259 | return {
260 | "duration_minutes": actual_duration / 60,
261 | "total_operations": total_operations,
262 | "total_errors": total_errors,
263 | "error_rate": total_errors / total_operations
264 | if total_operations > 0
265 | else 0,
266 | "operations_per_minute": total_operations / (actual_duration / 60),
267 | "avg_operation_time": sum(operation_times) / len(operation_times)
268 | if operation_times
269 | else 0,
270 | "resource_trends": trend_analysis,
271 | "concurrent_load": concurrent_load,
272 | }
273 |
274 | async def memory_leak_detection_test(
275 | self, iterations: int = 1000
276 | ) -> dict[str, Any]:
277 | """Test for memory leaks over many iterations."""
278 | logger.info(f"Starting memory leak detection test with {iterations} iterations")
279 |
280 | self.resource_monitor.start_monitoring()
281 | engine = VectorBTEngine(data_provider=self.data_provider)
282 |
283 | initial_memory = self.resource_monitor.get_current_snapshot().memory_rss_mb
284 | memory_measurements = []
285 |
286 | try:
287 | for i in range(iterations):
288 | # Run backtest operation
289 | symbol = f"LEAK_TEST_{i % 10}"
290 |
291 | await engine.run_backtest(
292 | symbol=symbol,
293 | strategy_type="sma_cross",
294 | parameters=STRATEGY_TEMPLATES["sma_cross"]["parameters"],
295 | start_date="2023-01-01",
296 | end_date="2023-12-31",
297 | )
298 |
299 | # Force garbage collection every 50 iterations
300 | if i % 50 == 0:
301 | gc.collect()
302 | snapshot = self.resource_monitor.get_current_snapshot()
303 | memory_measurements.append(
304 | {
305 | "iteration": i,
306 | "memory_mb": snapshot.memory_rss_mb,
307 | "memory_growth": snapshot.memory_rss_mb - initial_memory,
308 | }
309 | )
310 |
311 | if i % 200 == 0:
312 | logger.info(
313 | f"Iteration {i}: Memory = {snapshot.memory_rss_mb:.1f}MB "
314 | f"(+{snapshot.memory_rss_mb - initial_memory:.1f}MB)"
315 | )
316 |
317 | finally:
318 | self.resource_monitor.stop_monitoring()
319 |
320 | final_memory = self.resource_monitor.get_current_snapshot().memory_rss_mb
321 | total_growth = final_memory - initial_memory
322 |
323 | # Analyze memory leak pattern
324 | if len(memory_measurements) > 2:
325 | iterations_list = [m["iteration"] for m in memory_measurements]
326 | growth_list = [m["memory_growth"] for m in memory_measurements]
327 |
328 | # Linear regression to detect memory leak
329 | n = len(iterations_list)
330 | sum_x = sum(iterations_list)
331 | sum_y = sum(growth_list)
332 | sum_xy = sum(
333 | x * y for x, y in zip(iterations_list, growth_list, strict=False)
334 | )
335 | sum_xx = sum(x * x for x in iterations_list)
336 |
337 | if n * sum_xx != sum_x * sum_x:
338 | leak_rate = (n * sum_xy - sum_x * sum_y) / (n * sum_xx - sum_x * sum_x)
339 | else:
340 | leak_rate = 0
341 |
342 | # Memory leak per 1000 iterations
343 | leak_per_1000_iterations = leak_rate * 1000
344 | else:
345 | leak_rate = 0
346 | leak_per_1000_iterations = 0
347 |
348 | return {
349 | "iterations": iterations,
350 | "initial_memory_mb": initial_memory,
351 | "final_memory_mb": final_memory,
352 | "total_memory_growth_mb": total_growth,
353 | "leak_rate_mb_per_iteration": leak_rate,
354 | "leak_per_1000_iterations_mb": leak_per_1000_iterations,
355 | "memory_measurements": memory_measurements,
356 | "leak_detected": abs(leak_per_1000_iterations)
357 | > 10.0, # More than 10MB per 1000 iterations
358 | }
359 |
360 | async def cpu_stress_test(
361 | self, duration_minutes: int = 10, cpu_target: float = 0.9
362 | ) -> dict[str, Any]:
363 | """Test CPU utilization under stress."""
364 | logger.info(
365 | f"Starting CPU stress test: {duration_minutes} minutes at {cpu_target * 100}% target"
366 | )
367 |
368 | self.resource_monitor.start_monitoring()
369 |
370 | # Create CPU-intensive background load
371 | stop_event = threading.Event()
372 | cpu_threads = []
373 |
374 | def cpu_intensive_task():
375 | """CPU-intensive computation."""
376 | while not stop_event.is_set():
377 | # Perform CPU-intensive work
378 | for _ in range(10000):
379 | _ = sum(i**2 for i in range(100))
380 | time.sleep(0.001) # Brief pause
381 |
382 | try:
383 | # Start CPU load threads
384 | num_cpu_threads = max(1, int(psutil.cpu_count() * cpu_target))
385 | for _ in range(num_cpu_threads):
386 | thread = threading.Thread(target=cpu_intensive_task, daemon=True)
387 | thread.start()
388 | cpu_threads.append(thread)
389 |
390 | # Run backtests under CPU stress
391 | engine = VectorBTEngine(data_provider=self.data_provider)
392 | start_time = time.time()
393 | end_time = start_time + (duration_minutes * 60)
394 |
395 | operations_completed = 0
396 | cpu_stress_errors = 0
397 | response_times = []
398 |
399 | while time.time() < end_time:
400 | try:
401 | op_start = time.time()
402 |
403 | symbol = f"CPU_STRESS_{operations_completed % 5}"
404 | await asyncio.wait_for(
405 | engine.run_backtest(
406 | symbol=symbol,
407 | strategy_type="rsi",
408 | parameters=STRATEGY_TEMPLATES["rsi"]["parameters"],
409 | start_date="2023-01-01",
410 | end_date="2023-12-31",
411 | ),
412 | timeout=30.0, # Prevent hanging under CPU stress
413 | )
414 |
415 | response_time = time.time() - op_start
416 | response_times.append(response_time)
417 | operations_completed += 1
418 |
419 | except TimeoutError:
420 | cpu_stress_errors += 1
421 | logger.warning("Operation timed out under CPU stress")
422 |
423 | except Exception as e:
424 | cpu_stress_errors += 1
425 | logger.error(f"Operation failed under CPU stress: {e}")
426 |
427 | # Brief pause between operations
428 | await asyncio.sleep(1.0)
429 |
430 | finally:
431 | # Stop CPU stress
432 | stop_event.set()
433 | for thread in cpu_threads:
434 | thread.join(timeout=1.0)
435 |
436 | self.resource_monitor.stop_monitoring()
437 |
438 | trend_analysis = self.resource_monitor.analyze_trends()
439 |
440 | return {
441 | "duration_minutes": duration_minutes,
442 | "cpu_target_percent": cpu_target * 100,
443 | "operations_completed": operations_completed,
444 | "cpu_stress_errors": cpu_stress_errors,
445 | "error_rate": cpu_stress_errors / (operations_completed + cpu_stress_errors)
446 | if (operations_completed + cpu_stress_errors) > 0
447 | else 0,
448 | "avg_response_time": sum(response_times) / len(response_times)
449 | if response_times
450 | else 0,
451 | "max_response_time": max(response_times) if response_times else 0,
452 | "avg_cpu_utilization": trend_analysis["avg_cpu_percent"],
453 | "peak_cpu_utilization": trend_analysis["peak_cpu_percent"],
454 | }
455 |
456 | async def database_connection_exhaustion_test(
457 | self, db_session, max_connections: int = 50
458 | ) -> dict[str, Any]:
459 | """Test database behavior under connection exhaustion."""
460 | logger.info(
461 | f"Starting database connection exhaustion test with {max_connections} connections"
462 | )
463 |
464 | # Generate test data
465 | engine = VectorBTEngine(data_provider=self.data_provider)
466 | test_results = []
467 |
468 | for i in range(5):
469 | result = await engine.run_backtest(
470 | symbol=f"DB_EXHAUST_{i}",
471 | strategy_type="macd",
472 | parameters=STRATEGY_TEMPLATES["macd"]["parameters"],
473 | start_date="2023-01-01",
474 | end_date="2023-12-31",
475 | )
476 | test_results.append(result)
477 |
478 | # Test connection exhaustion
479 | async def database_operation(conn_id: int) -> dict[str, Any]:
480 | """Single database operation holding connection."""
481 | try:
482 | with BacktestPersistenceManager(session=db_session) as persistence:
483 | # Hold connection and perform operations
484 | saved_ids = []
485 |
486 | for result in test_results:
487 | backtest_id = persistence.save_backtest_result(
488 | vectorbt_results=result,
489 | execution_time=2.0,
490 | notes=f"Connection exhaustion test {conn_id}",
491 | )
492 | saved_ids.append(backtest_id)
493 |
494 | # Perform queries
495 | for backtest_id in saved_ids:
496 | persistence.get_backtest_by_id(backtest_id)
497 |
498 | # Hold connection for some time
499 | await asyncio.sleep(2.0)
500 |
501 | return {
502 | "connection_id": conn_id,
503 | "operations_completed": len(saved_ids) * 2, # Save + retrieve
504 | "success": True,
505 | }
506 |
507 | except Exception as e:
508 | return {
509 | "connection_id": conn_id,
510 | "error": str(e),
511 | "success": False,
512 | }
513 |
514 | # Create many concurrent database operations
515 | start_time = time.time()
516 |
517 | connection_tasks = [database_operation(i) for i in range(max_connections)]
518 |
519 | # Execute with timeout to prevent hanging
520 | try:
521 | results = await asyncio.wait_for(
522 | asyncio.gather(*connection_tasks, return_exceptions=True), timeout=60.0
523 | )
524 | except TimeoutError:
525 | logger.warning("Database connection test timed out")
526 | results = []
527 |
528 | execution_time = time.time() - start_time
529 |
530 | # Analyze results
531 | successful_connections = sum(
532 | 1 for r in results if isinstance(r, dict) and r.get("success", False)
533 | )
534 | failed_connections = len(results) - successful_connections
535 |
536 | total_operations = sum(
537 | r.get("operations_completed", 0)
538 | for r in results
539 | if isinstance(r, dict) and r.get("success", False)
540 | )
541 |
542 | return {
543 | "max_connections_attempted": max_connections,
544 | "successful_connections": successful_connections,
545 | "failed_connections": failed_connections,
546 | "connection_success_rate": successful_connections / max_connections
547 | if max_connections > 0
548 | else 0,
549 | "total_operations": total_operations,
550 | "execution_time": execution_time,
551 | "operations_per_second": total_operations / execution_time
552 | if execution_time > 0
553 | else 0,
554 | }
555 |
556 | async def file_descriptor_exhaustion_test(self) -> dict[str, Any]:
557 | """Test file descriptor usage patterns."""
558 | logger.info("Starting file descriptor exhaustion test")
559 |
560 | initial_snapshot = self.resource_monitor.get_current_snapshot()
561 | initial_fds = initial_snapshot.file_descriptors
562 |
563 | self.resource_monitor.start_monitoring()
564 |
565 | # Get system file descriptor limit
566 | try:
567 | soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
568 | except Exception:
569 | soft_limit, hard_limit = 1024, 4096 # Default assumptions
570 |
571 | logger.info(
572 | f"FD limits - Soft: {soft_limit}, Hard: {hard_limit}, Initial: {initial_fds}"
573 | )
574 |
575 | try:
576 | engine = VectorBTEngine(data_provider=self.data_provider)
577 |
578 | # Run many operations to stress file descriptor usage
579 | fd_measurements = []
580 | max_operations = min(100, soft_limit // 10) # Conservative approach
581 |
582 | for i in range(max_operations):
583 | await engine.run_backtest(
584 | symbol=f"FD_TEST_{i}",
585 | strategy_type="sma_cross",
586 | parameters=STRATEGY_TEMPLATES["sma_cross"]["parameters"],
587 | start_date="2023-01-01",
588 | end_date="2023-12-31",
589 | )
590 |
591 | if i % 10 == 0:
592 | snapshot = self.resource_monitor.get_current_snapshot()
593 | fd_measurements.append(
594 | {
595 | "iteration": i,
596 | "file_descriptors": snapshot.file_descriptors,
597 | "fd_growth": snapshot.file_descriptors - initial_fds,
598 | }
599 | )
600 |
601 | if snapshot.file_descriptors > soft_limit * 0.8:
602 | logger.warning(
603 | f"High FD usage detected: {snapshot.file_descriptors}/{soft_limit}"
604 | )
605 |
606 | finally:
607 | self.resource_monitor.stop_monitoring()
608 |
609 | final_snapshot = self.resource_monitor.get_current_snapshot()
610 | final_fds = final_snapshot.file_descriptors
611 | fd_growth = final_fds - initial_fds
612 |
613 | # Analyze FD usage pattern
614 | peak_fds = max([m["file_descriptors"] for m in fd_measurements] + [final_fds])
615 | fd_utilization = peak_fds / soft_limit if soft_limit > 0 else 0
616 |
617 | return {
618 | "initial_file_descriptors": initial_fds,
619 | "final_file_descriptors": final_fds,
620 | "peak_file_descriptors": peak_fds,
621 | "fd_growth": fd_growth,
622 | "soft_limit": soft_limit,
623 | "hard_limit": hard_limit,
624 | "fd_utilization_percent": fd_utilization * 100,
625 | "fd_measurements": fd_measurements,
626 | "operations_completed": max_operations,
627 | }
628 |
629 |
630 | class TestStressTesting:
631 | """Stress testing suite."""
632 |
633 | @pytest.fixture
634 | async def stress_data_provider(self):
635 | """Create data provider optimized for stress testing."""
636 | provider = Mock()
637 |
638 | # Cache to reduce computation overhead during stress tests
639 | data_cache = {}
640 |
641 | def get_stress_test_data(symbol: str) -> pd.DataFrame:
642 | """Get cached data for stress testing."""
643 | if symbol not in data_cache:
644 | # Generate smaller dataset for faster stress testing
645 | dates = pd.date_range(start="2023-06-01", end="2023-12-31", freq="D")
646 | seed = hash(symbol) % 1000
647 | np.random.seed(seed)
648 |
649 | returns = np.random.normal(0.001, 0.02, len(dates))
650 | prices = 100 * np.cumprod(1 + returns)
651 |
652 | data_cache[symbol] = pd.DataFrame(
653 | {
654 | "Open": prices * np.random.uniform(0.99, 1.01, len(dates)),
655 | "High": prices * np.random.uniform(1.01, 1.03, len(dates)),
656 | "Low": prices * np.random.uniform(0.97, 0.99, len(dates)),
657 | "Close": prices,
658 | "Volume": np.random.randint(1000000, 5000000, len(dates)),
659 | "Adj Close": prices,
660 | },
661 | index=dates,
662 | )
663 |
664 | return data_cache[symbol].copy()
665 |
666 | provider.get_stock_data.side_effect = get_stress_test_data
667 | return provider
668 |
669 | @pytest.mark.slow
670 | async def test_sustained_load_15_minutes(self, stress_data_provider):
671 | """Test sustained load for 15 minutes (abbreviated from 1 hour for CI)."""
672 | stress_runner = StressTestRunner(stress_data_provider)
673 |
674 | result = await stress_runner.sustained_load_test(
675 | duration_minutes=15, # Reduced for CI/testing
676 | concurrent_load=8,
677 | )
678 |
679 | # Assertions for sustained load
680 | assert result["error_rate"] <= 0.05, (
681 | f"Error rate too high: {result['error_rate']:.3f}"
682 | )
683 | assert result["operations_per_minute"] >= 10, (
684 | f"Throughput too low: {result['operations_per_minute']:.1f} ops/min"
685 | )
686 |
687 | # Resource growth should be reasonable
688 | trends = result["resource_trends"]
689 | assert trends["memory_growth_rate_mb_per_hour"] <= 100, (
690 | f"Memory growth rate too high: {trends['memory_growth_rate_mb_per_hour']:.1f} MB/hour"
691 | )
692 |
693 | logger.info(
694 | f"✓ Sustained load test completed: {result['total_operations']} operations in {result['duration_minutes']:.1f} minutes"
695 | )
696 | return result
697 |
698 | async def test_memory_leak_detection(self, stress_data_provider):
699 | """Test for memory leaks over many iterations."""
700 | stress_runner = StressTestRunner(stress_data_provider)
701 |
702 | result = await stress_runner.memory_leak_detection_test(iterations=200)
703 |
704 | # Memory leak assertions
705 | assert not result["leak_detected"], (
706 | f"Memory leak detected: {result['leak_per_1000_iterations_mb']:.2f} MB per 1000 iterations"
707 | )
708 | assert result["total_memory_growth_mb"] <= 300, (
709 | f"Total memory growth too high: {result['total_memory_growth_mb']:.1f} MB"
710 | )
711 |
712 | logger.info(
713 | f"✓ Memory leak test completed: {result['total_memory_growth_mb']:.1f}MB growth over {result['iterations']} iterations"
714 | )
715 | return result
716 |
717 | async def test_cpu_stress_resilience(self, stress_data_provider):
718 | """Test system resilience under CPU stress."""
719 | stress_runner = StressTestRunner(stress_data_provider)
720 |
721 | result = await stress_runner.cpu_stress_test(
722 | duration_minutes=5, # Reduced for testing
723 | cpu_target=0.7, # 70% CPU utilization
724 | )
725 |
726 | # CPU stress assertions
727 | assert result["error_rate"] <= 0.2, (
728 | f"Error rate too high under CPU stress: {result['error_rate']:.3f}"
729 | )
730 | assert result["avg_response_time"] <= 10.0, (
731 | f"Response time too slow under CPU stress: {result['avg_response_time']:.2f}s"
732 | )
733 | assert result["operations_completed"] >= 10, (
734 | f"Too few operations completed: {result['operations_completed']}"
735 | )
736 |
737 | logger.info(
738 | f"✓ CPU stress test completed: {result['operations_completed']} operations with {result['avg_cpu_utilization']:.1f}% avg CPU"
739 | )
740 | return result
741 |
742 | async def test_database_connection_stress(self, stress_data_provider, db_session):
743 | """Test database performance under connection stress."""
744 | stress_runner = StressTestRunner(stress_data_provider)
745 |
746 | result = await stress_runner.database_connection_exhaustion_test(
747 | db_session=db_session,
748 | max_connections=20, # Reduced for testing
749 | )
750 |
751 | # Database stress assertions
752 | assert result["connection_success_rate"] >= 0.8, (
753 | f"Connection success rate too low: {result['connection_success_rate']:.3f}"
754 | )
755 | assert result["operations_per_second"] >= 5.0, (
756 | f"Database throughput too low: {result['operations_per_second']:.2f} ops/s"
757 | )
758 |
759 | logger.info(
760 | f"✓ Database stress test completed: {result['successful_connections']}/{result['max_connections_attempted']} connections succeeded"
761 | )
762 | return result
763 |
764 | async def test_file_descriptor_management(self, stress_data_provider):
765 | """Test file descriptor usage under stress."""
766 | stress_runner = StressTestRunner(stress_data_provider)
767 |
768 | result = await stress_runner.file_descriptor_exhaustion_test()
769 |
770 | # File descriptor assertions
771 | assert result["fd_utilization_percent"] <= 50.0, (
772 | f"FD utilization too high: {result['fd_utilization_percent']:.1f}%"
773 | )
774 | assert result["fd_growth"] <= 100, f"FD growth too high: {result['fd_growth']}"
775 |
776 | logger.info(
777 | f"✓ File descriptor test completed: {result['peak_file_descriptors']} peak FDs ({result['fd_utilization_percent']:.1f}% utilization)"
778 | )
779 | return result
780 |
781 | async def test_queue_overflow_scenarios(self, stress_data_provider):
782 | """Test queue management under overflow conditions."""
783 | # Simulate queue overflow by creating more tasks than can be processed
784 | max_queue_size = 50
785 | overflow_tasks = 100
786 |
787 | semaphore = asyncio.Semaphore(5) # Limit concurrent processing
788 | processed_tasks = 0
789 | overflow_errors = 0
790 |
791 | async def queue_task(task_id: int):
792 | nonlocal processed_tasks, overflow_errors
793 |
794 | try:
795 | async with semaphore:
796 | engine = VectorBTEngine(data_provider=stress_data_provider)
797 |
798 | await engine.run_backtest(
799 | symbol=f"QUEUE_{task_id % 10}",
800 | strategy_type="sma_cross",
801 | parameters=STRATEGY_TEMPLATES["sma_cross"]["parameters"],
802 | start_date="2023-06-01",
803 | end_date="2023-12-31",
804 | )
805 |
806 | processed_tasks += 1
807 |
808 | except Exception as e:
809 | overflow_errors += 1
810 | logger.error(f"Queue task {task_id} failed: {e}")
811 |
812 | # Create tasks faster than they can be processed
813 | start_time = time.time()
814 |
815 | tasks = []
816 | for i in range(overflow_tasks):
817 | task = asyncio.create_task(queue_task(i))
818 | tasks.append(task)
819 |
820 | # Create tasks rapidly to test queue management
821 | if i < max_queue_size:
822 | await asyncio.sleep(0.01) # Rapid creation
823 | else:
824 | await asyncio.sleep(0.1) # Slower creation after queue fills
825 |
826 | # Wait for all tasks to complete
827 | await asyncio.gather(*tasks, return_exceptions=True)
828 |
829 | execution_time = time.time() - start_time
830 |
831 | # Queue overflow assertions
832 | processing_success_rate = processed_tasks / overflow_tasks
833 | assert processing_success_rate >= 0.8, (
834 | f"Queue processing success rate too low: {processing_success_rate:.3f}"
835 | )
836 | assert execution_time < 120.0, (
837 | f"Queue processing took too long: {execution_time:.1f}s"
838 | )
839 |
840 | logger.info(
841 | f"✓ Queue overflow test completed: {processed_tasks}/{overflow_tasks} tasks processed in {execution_time:.1f}s"
842 | )
843 |
844 | return {
845 | "overflow_tasks": overflow_tasks,
846 | "processed_tasks": processed_tasks,
847 | "overflow_errors": overflow_errors,
848 | "processing_success_rate": processing_success_rate,
849 | "execution_time": execution_time,
850 | }
851 |
852 | async def test_comprehensive_stress_suite(self, stress_data_provider, db_session):
853 | """Run comprehensive stress testing suite."""
854 | logger.info("Starting Comprehensive Stress Testing Suite...")
855 |
856 | stress_results = {}
857 |
858 | # Run individual stress tests
859 | stress_results["sustained_load"] = await self.test_sustained_load_15_minutes(
860 | stress_data_provider
861 | )
862 | stress_results["memory_leak"] = await self.test_memory_leak_detection(
863 | stress_data_provider
864 | )
865 | stress_results["cpu_stress"] = await self.test_cpu_stress_resilience(
866 | stress_data_provider
867 | )
868 | stress_results["database_stress"] = await self.test_database_connection_stress(
869 | stress_data_provider, db_session
870 | )
871 | stress_results["file_descriptors"] = await self.test_file_descriptor_management(
872 | stress_data_provider
873 | )
874 | stress_results["queue_overflow"] = await self.test_queue_overflow_scenarios(
875 | stress_data_provider
876 | )
877 |
878 | # Aggregate stress test analysis
879 | total_tests = len(stress_results)
880 | passed_tests = 0
881 | critical_failures = []
882 |
883 | for test_name, result in stress_results.items():
884 | # Simple pass/fail based on whether test completed without major issues
885 | test_passed = True
886 |
887 | if test_name == "sustained_load" and result["error_rate"] > 0.1:
888 | test_passed = False
889 | critical_failures.append(
890 | f"Sustained load error rate: {result['error_rate']:.3f}"
891 | )
892 | elif test_name == "memory_leak" and result["leak_detected"]:
893 | test_passed = False
894 | critical_failures.append(
895 | f"Memory leak detected: {result['leak_per_1000_iterations_mb']:.2f} MB/1k iterations"
896 | )
897 | elif test_name == "cpu_stress" and result["error_rate"] > 0.3:
898 | test_passed = False
899 | critical_failures.append(
900 | f"CPU stress error rate: {result['error_rate']:.3f}"
901 | )
902 |
903 | if test_passed:
904 | passed_tests += 1
905 |
906 | overall_pass_rate = passed_tests / total_tests
907 |
908 | logger.info(
909 | f"\n{'=' * 60}\n"
910 | f"COMPREHENSIVE STRESS TEST REPORT\n"
911 | f"{'=' * 60}\n"
912 | f"Total Tests: {total_tests}\n"
913 | f"Passed: {passed_tests}\n"
914 | f"Overall Pass Rate: {overall_pass_rate:.1%}\n"
915 | f"Critical Failures: {len(critical_failures)}\n"
916 | f"{'=' * 60}\n"
917 | )
918 |
919 | # Assert overall stress test success
920 | assert overall_pass_rate >= 0.8, (
921 | f"Overall stress test pass rate too low: {overall_pass_rate:.1%}"
922 | )
923 | assert len(critical_failures) <= 1, (
924 | f"Too many critical failures: {critical_failures}"
925 | )
926 |
927 | return {
928 | "overall_pass_rate": overall_pass_rate,
929 | "critical_failures": critical_failures,
930 | "stress_results": stress_results,
931 | }
932 |
933 |
934 | if __name__ == "__main__":
935 | # Run stress testing suite
936 | pytest.main(
937 | [
938 | __file__,
939 | "-v",
940 | "--tb=short",
941 | "--asyncio-mode=auto",
942 | "--timeout=1800", # 30 minute timeout for stress tests
943 | "-m",
944 | "not slow", # Skip slow tests by default
945 | ]
946 | )
947 |
```
--------------------------------------------------------------------------------
/tests/test_database_pool_config.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Comprehensive tests for DatabasePoolConfig.
3 |
4 | This module tests the enhanced database pool configuration that provides
5 | validation, monitoring, and optimization capabilities. Tests cover:
6 | - Pool validation logic against database limits
7 | - Warning conditions for insufficient pool sizing
8 | - Environment variable overrides
9 | - Factory methods (development, production, high-concurrency)
10 | - Monitoring thresholds and SQLAlchemy event listeners
11 | - Integration with existing DatabaseConfig
12 | - Production validation checks
13 | """
14 |
15 | import os
16 | import warnings
17 | from unittest.mock import Mock, patch
18 |
19 | import pytest
20 | from sqlalchemy.pool import QueuePool
21 |
22 | from maverick_mcp.config.database import (
23 | DatabasePoolConfig,
24 | create_engine_with_enhanced_config,
25 | create_monitored_engine_kwargs,
26 | get_default_pool_config,
27 | get_development_pool_config,
28 | get_high_concurrency_pool_config,
29 | get_pool_config_from_settings,
30 | validate_production_config,
31 | )
32 | from maverick_mcp.providers.interfaces.persistence import DatabaseConfig
33 |
34 |
35 | class TestDatabasePoolConfig:
36 | """Test the main DatabasePoolConfig class."""
37 |
38 | def test_default_configuration(self):
39 | """Test default configuration values."""
40 | config = DatabasePoolConfig()
41 |
42 | # Test environment variable defaults
43 | assert config.pool_size == int(os.getenv("DB_POOL_SIZE", "20"))
44 | assert config.max_overflow == int(os.getenv("DB_MAX_OVERFLOW", "10"))
45 | assert config.pool_timeout == int(os.getenv("DB_POOL_TIMEOUT", "30"))
46 | assert config.pool_recycle == int(os.getenv("DB_POOL_RECYCLE", "3600"))
47 | assert config.max_database_connections == int(
48 | os.getenv("DB_MAX_CONNECTIONS", "100")
49 | )
50 | assert config.reserved_superuser_connections == int(
51 | os.getenv("DB_RESERVED_SUPERUSER_CONNECTIONS", "3")
52 | )
53 | assert config.expected_concurrent_users == int(
54 | os.getenv("DB_EXPECTED_CONCURRENT_USERS", "20")
55 | )
56 | assert config.connections_per_user == float(
57 | os.getenv("DB_CONNECTIONS_PER_USER", "1.2")
58 | )
59 | assert config.pool_pre_ping == (
60 | os.getenv("DB_POOL_PRE_PING", "true").lower() == "true"
61 | )
62 | assert config.echo_pool == (
63 | os.getenv("DB_ECHO_POOL", "false").lower() == "true"
64 | )
65 |
66 | @patch.dict(
67 | os.environ,
68 | {
69 | "DB_POOL_SIZE": "25",
70 | "DB_MAX_OVERFLOW": "10",
71 | "DB_POOL_TIMEOUT": "45",
72 | "DB_POOL_RECYCLE": "1800",
73 | "DB_MAX_CONNECTIONS": "80",
74 | "DB_RESERVED_SUPERUSER_CONNECTIONS": "2",
75 | "DB_EXPECTED_CONCURRENT_USERS": "25",
76 | "DB_CONNECTIONS_PER_USER": "1.2",
77 | "DB_POOL_PRE_PING": "false",
78 | "DB_ECHO_POOL": "true",
79 | },
80 | )
81 | def test_environment_variable_overrides(self):
82 | """Test that environment variables override defaults."""
83 | config = DatabasePoolConfig()
84 |
85 | assert config.pool_size == 25
86 | assert config.max_overflow == 10
87 | assert config.pool_timeout == 45
88 | assert config.pool_recycle == 1800
89 | assert config.max_database_connections == 80
90 | assert config.reserved_superuser_connections == 2
91 | assert config.expected_concurrent_users == 25
92 | assert config.connections_per_user == 1.2
93 | assert config.pool_pre_ping is False
94 | assert config.echo_pool is True
95 |
96 | def test_valid_configuration(self):
97 | """Test a valid configuration passes validation."""
98 | config = DatabasePoolConfig(
99 | pool_size=10,
100 | max_overflow=5,
101 | max_database_connections=50,
102 | reserved_superuser_connections=3,
103 | expected_concurrent_users=10,
104 | connections_per_user=1.2,
105 | )
106 |
107 | # Should not raise any exceptions
108 | assert config.pool_size == 10
109 | assert config.max_overflow == 5
110 |
111 | # Calculated values
112 | total_app_connections = config.pool_size + config.max_overflow
113 | available_connections = (
114 | config.max_database_connections - config.reserved_superuser_connections
115 | )
116 | assert total_app_connections <= available_connections
117 |
118 | def test_validation_exceeds_database_capacity(self):
119 | """Test validation failure when pool exceeds database capacity."""
120 | with pytest.raises(
121 | ValueError, match="Pool configuration exceeds database capacity"
122 | ):
123 | DatabasePoolConfig(
124 | pool_size=50,
125 | max_overflow=30, # Total = 80
126 | max_database_connections=70, # Available = 67 (70-3)
127 | reserved_superuser_connections=3,
128 | )
129 |
130 | def test_validation_insufficient_for_expected_load(self):
131 | """Test validation failure when pool is insufficient for expected load."""
132 | with pytest.raises(
133 | ValueError, match="Total connection capacity .* is insufficient"
134 | ):
135 | DatabasePoolConfig(
136 | pool_size=5,
137 | max_overflow=0, # Total capacity = 5
138 | expected_concurrent_users=10,
139 | connections_per_user=1.0, # Expected demand = 10
140 | max_database_connections=50,
141 | )
142 |
143 | def test_validation_warning_for_small_pool(self):
144 | """Test warning when pool size may be insufficient."""
145 | with warnings.catch_warnings(record=True) as w:
146 | warnings.simplefilter("always")
147 |
148 | DatabasePoolConfig(
149 | pool_size=5, # Small pool
150 | max_overflow=15, # But enough overflow to meet demand
151 | expected_concurrent_users=10,
152 | connections_per_user=1.5, # Expected demand = 15
153 | max_database_connections=50,
154 | )
155 |
156 | # Should generate a warning
157 | assert len(w) > 0
158 | assert "Pool size (5) may be insufficient" in str(w[0].message)
159 |
160 | def test_field_validation_ranges(self):
161 | """Test field validation for valid ranges."""
162 | from pydantic import ValidationError
163 |
164 | # Test valid ranges with proper expected demand
165 | config = DatabasePoolConfig(
166 | pool_size=5, # Minimum safe size
167 | max_overflow=0, # Minimum
168 | pool_timeout=1, # Minimum
169 | pool_recycle=300, # Minimum
170 | expected_concurrent_users=3, # Lower expected demand
171 | connections_per_user=1.0,
172 | )
173 | assert config.pool_size == 5
174 |
175 | config = DatabasePoolConfig(
176 | pool_size=80, # Large but fits in database capacity
177 | max_overflow=15, # Reasonable overflow
178 | pool_timeout=300, # Maximum
179 | pool_recycle=7200, # Maximum
180 | expected_concurrent_users=85, # Fit within total capacity of 95
181 | connections_per_user=1.0,
182 | max_database_connections=120, # Higher limit to accommodate pool
183 | )
184 | assert config.pool_size == 80
185 |
186 | # Test invalid ranges
187 | with pytest.raises(ValidationError):
188 | DatabasePoolConfig(pool_size=0) # Below minimum
189 |
190 | with pytest.raises(ValidationError):
191 | DatabasePoolConfig(pool_size=101) # Above maximum
192 |
193 | with pytest.raises(ValidationError):
194 | DatabasePoolConfig(max_overflow=-1) # Below minimum
195 |
196 | with pytest.raises(ValidationError):
197 | DatabasePoolConfig(max_overflow=51) # Above maximum
198 |
199 | def test_get_pool_kwargs(self):
200 | """Test SQLAlchemy pool configuration generation."""
201 | config = DatabasePoolConfig(
202 | pool_size=15,
203 | max_overflow=8,
204 | pool_timeout=45,
205 | pool_recycle=1800,
206 | pool_pre_ping=True,
207 | echo_pool=True,
208 | expected_concurrent_users=18, # Match capacity
209 | connections_per_user=1.0,
210 | )
211 |
212 | kwargs = config.get_pool_kwargs()
213 |
214 | expected = {
215 | "poolclass": QueuePool,
216 | "pool_size": 15,
217 | "max_overflow": 8,
218 | "pool_timeout": 45,
219 | "pool_recycle": 1800,
220 | "pool_pre_ping": True,
221 | "echo_pool": True,
222 | }
223 |
224 | assert kwargs == expected
225 |
226 | def test_get_monitoring_thresholds(self):
227 | """Test monitoring threshold calculation."""
228 | config = DatabasePoolConfig(pool_size=20, max_overflow=10)
229 | thresholds = config.get_monitoring_thresholds()
230 |
231 | expected = {
232 | "warning_threshold": int(20 * 0.8), # 16
233 | "critical_threshold": int(20 * 0.95), # 19
234 | "pool_size": 20,
235 | "max_overflow": 10,
236 | "total_capacity": 30,
237 | }
238 |
239 | assert thresholds == expected
240 |
241 | def test_validate_against_database_limits_matching(self):
242 | """Test validation when actual limits match configuration."""
243 | config = DatabasePoolConfig(max_database_connections=100)
244 |
245 | # Should not raise any exceptions when limits match
246 | config.validate_against_database_limits(100)
247 | assert config.max_database_connections == 100
248 |
249 | def test_validate_against_database_limits_higher_actual(self):
250 | """Test validation when actual limits are higher."""
251 | config = DatabasePoolConfig(max_database_connections=100)
252 |
253 | with patch("maverick_mcp.config.database.logger") as mock_logger:
254 | config.validate_against_database_limits(150)
255 |
256 | # Should update configuration and log info
257 | assert config.max_database_connections == 150
258 | mock_logger.info.assert_called_once()
259 |
260 | def test_validate_against_database_limits_lower_actual_safe(self):
261 | """Test validation when actual limits are lower but pool still fits."""
262 | config = DatabasePoolConfig(
263 | pool_size=10,
264 | max_overflow=5, # Total = 15
265 | max_database_connections=100,
266 | reserved_superuser_connections=3,
267 | expected_concurrent_users=12, # Fit within total capacity of 15
268 | connections_per_user=1.0,
269 | )
270 |
271 | with patch("maverick_mcp.config.database.logger") as mock_logger:
272 | # Actual limit is 80, available is 77, pool needs 15 - should be fine
273 | config.validate_against_database_limits(80)
274 |
275 | mock_logger.warning.assert_called_once()
276 | warning_call = mock_logger.warning.call_args[0][0]
277 | assert "lower than configured" in warning_call
278 |
279 | def test_validate_against_database_limits_lower_actual_unsafe(self):
280 | """Test validation failure when actual limits are too low."""
281 | config = DatabasePoolConfig(
282 | pool_size=30,
283 | max_overflow=20, # Total = 50
284 | max_database_connections=100,
285 | reserved_superuser_connections=3,
286 | )
287 |
288 | with pytest.raises(
289 | ValueError, match="Configuration invalid for actual database limits"
290 | ):
291 | # Actual limit is 40, available is 37, pool needs 50 - should fail
292 | config.validate_against_database_limits(40)
293 |
294 | def test_to_legacy_config(self):
295 | """Test conversion to legacy DatabaseConfig."""
296 | config = DatabasePoolConfig(
297 | pool_size=15,
298 | max_overflow=8,
299 | pool_timeout=45,
300 | pool_recycle=1800,
301 | echo_pool=True,
302 | expected_concurrent_users=18, # Fit within total capacity of 23
303 | connections_per_user=1.0,
304 | )
305 |
306 | database_url = "postgresql://user:pass@localhost/test"
307 | legacy_config = config.to_legacy_config(database_url)
308 |
309 | assert isinstance(legacy_config, DatabaseConfig)
310 | assert legacy_config.database_url == database_url
311 | assert legacy_config.pool_size == 15
312 | assert legacy_config.max_overflow == 8
313 | assert legacy_config.pool_timeout == 45
314 | assert legacy_config.pool_recycle == 1800
315 | assert legacy_config.echo is True
316 | assert legacy_config.autocommit is False
317 | assert legacy_config.autoflush is True
318 | assert legacy_config.expire_on_commit is True
319 |
320 | def test_from_legacy_config(self):
321 | """Test creation from legacy DatabaseConfig."""
322 | legacy_config = DatabaseConfig(
323 | database_url="postgresql://user:pass@localhost/test",
324 | pool_size=12,
325 | max_overflow=6,
326 | pool_timeout=60,
327 | pool_recycle=2400,
328 | echo=False,
329 | )
330 |
331 | enhanced_config = DatabasePoolConfig.from_legacy_config(
332 | legacy_config,
333 | expected_concurrent_users=15, # Override
334 | max_database_connections=80, # Override
335 | )
336 |
337 | assert enhanced_config.pool_size == 12
338 | assert enhanced_config.max_overflow == 6
339 | assert enhanced_config.pool_timeout == 60
340 | assert enhanced_config.pool_recycle == 2400
341 | assert enhanced_config.echo_pool is False
342 | assert enhanced_config.expected_concurrent_users == 15 # Override applied
343 | assert enhanced_config.max_database_connections == 80 # Override applied
344 |
345 | def test_setup_pool_monitoring(self):
346 | """Test SQLAlchemy event listener setup."""
347 | config = DatabasePoolConfig(
348 | pool_size=10,
349 | echo_pool=True,
350 | expected_concurrent_users=15, # Fit within capacity
351 | connections_per_user=1.0,
352 | )
353 |
354 | # Create a mock engine with pool
355 | mock_engine = Mock()
356 | mock_pool = Mock()
357 | mock_pool.checkedout.return_value = 5
358 | mock_pool.checkedin.return_value = 3
359 | mock_engine.pool = mock_pool
360 |
361 | # Mock the event listener registration
362 | with patch("maverick_mcp.config.database.event") as mock_event:
363 | config.setup_pool_monitoring(mock_engine)
364 |
365 | # Verify event listeners were registered
366 | assert (
367 | mock_event.listens_for.call_count == 5
368 | ) # connect, checkout, checkin, invalidate, soft_invalidate
369 |
370 | # Test the event listener functions were called correctly
371 | expected_events = [
372 | "connect",
373 | "checkout",
374 | "checkin",
375 | "invalidate",
376 | "soft_invalidate",
377 | ]
378 | for call_args in mock_event.listens_for.call_args_list:
379 | assert call_args[0][0] is mock_engine
380 | assert call_args[0][1] in expected_events
381 |
382 |
383 | class TestFactoryFunctions:
384 | """Test factory functions for different configuration types."""
385 |
386 | def test_get_default_pool_config(self):
387 | """Test default pool configuration factory."""
388 | config = get_default_pool_config()
389 |
390 | assert isinstance(config, DatabasePoolConfig)
391 | # Should use environment variable defaults
392 | assert config.pool_size == int(os.getenv("DB_POOL_SIZE", "20"))
393 |
394 | def test_get_development_pool_config(self):
395 | """Test development pool configuration factory."""
396 | config = get_development_pool_config()
397 |
398 | assert isinstance(config, DatabasePoolConfig)
399 | assert config.pool_size == 5
400 | assert config.max_overflow == 2
401 | assert config.pool_timeout == 30
402 | assert config.pool_recycle == 3600
403 | assert config.expected_concurrent_users == 5
404 | assert config.connections_per_user == 1.0
405 | assert config.max_database_connections == 20
406 | assert config.reserved_superuser_connections == 2
407 | assert config.echo_pool is True # Debug enabled in development
408 |
409 | def test_get_high_concurrency_pool_config(self):
410 | """Test high concurrency pool configuration factory."""
411 | config = get_high_concurrency_pool_config()
412 |
413 | assert isinstance(config, DatabasePoolConfig)
414 | assert config.pool_size == 50
415 | assert config.max_overflow == 30
416 | assert config.pool_timeout == 60
417 | assert config.pool_recycle == 1800 # 30 minutes
418 | assert config.expected_concurrent_users == 60
419 | assert config.connections_per_user == 1.3
420 | assert config.max_database_connections == 200
421 | assert config.reserved_superuser_connections == 5
422 |
423 | def test_get_pool_config_from_settings_development(self):
424 | """Test configuration from settings in development."""
425 | # Create a mock settings module and settings object
426 | mock_settings_module = Mock()
427 | mock_settings_obj = Mock()
428 | mock_settings_obj.environment = "development"
429 | # Configure hasattr to return False for 'db' to avoid overrides path
430 | mock_settings_obj.configure_mock(**{"db": None})
431 | mock_settings_module.settings = mock_settings_obj
432 |
433 | # Patch the import to return our mock
434 | with patch.dict(
435 | "sys.modules", {"maverick_mcp.config.settings": mock_settings_module}
436 | ):
437 | # Also patch hasattr to return False for the db attribute
438 | with patch("builtins.hasattr", side_effect=lambda obj, attr: attr != "db"):
439 | config = get_pool_config_from_settings()
440 |
441 | # Should return development configuration
442 | assert config.pool_size == 5 # Development default
443 | assert config.echo_pool is True
444 |
445 | def test_get_pool_config_from_settings_production(self):
446 | """Test configuration from settings in production."""
447 | # Create a mock settings module and settings object
448 | mock_settings_module = Mock()
449 | mock_settings_obj = Mock()
450 | mock_settings_obj.environment = "production"
451 | mock_settings_module.settings = mock_settings_obj
452 |
453 | # Patch the import to return our mock
454 | with patch.dict(
455 | "sys.modules", {"maverick_mcp.config.settings": mock_settings_module}
456 | ):
457 | # Also patch hasattr to return False for the db attribute
458 | with patch("builtins.hasattr", side_effect=lambda obj, attr: attr != "db"):
459 | config = get_pool_config_from_settings()
460 |
461 | # Should return high concurrency configuration
462 | assert config.pool_size == 50 # Production default
463 | assert config.max_overflow == 30
464 |
465 | def test_get_pool_config_from_settings_with_overrides(self):
466 | """Test configuration from settings with database-specific overrides."""
467 | # Create a mock settings module and settings object
468 | mock_settings_module = Mock()
469 | mock_settings_obj = Mock()
470 | mock_settings_obj.environment = "development"
471 |
472 | # Create proper mock for db settings with real values, not Mock objects
473 | class MockDbSettings:
474 | pool_size = 8
475 | pool_max_overflow = 3
476 | pool_timeout = 60
477 |
478 | mock_settings_obj.db = MockDbSettings()
479 | mock_settings_module.settings = mock_settings_obj
480 |
481 | # Patch the import to return our mock
482 | with patch.dict(
483 | "sys.modules", {"maverick_mcp.config.settings": mock_settings_module}
484 | ):
485 | config = get_pool_config_from_settings()
486 |
487 | # Should use overrides
488 | assert config.pool_size == 8
489 | assert config.max_overflow == 3
490 | assert config.pool_timeout == 60
491 | # Other development defaults should remain
492 | assert config.echo_pool is True
493 |
494 | def test_get_pool_config_from_settings_import_error(self):
495 | """Test fallback when settings import fails."""
496 |
497 | # Create a mock import function that raises ImportError for settings module
498 | def mock_import(name, *args, **kwargs):
499 | if name == "maverick_mcp.config.settings":
500 | raise ImportError("No module named 'maverick_mcp.config.settings'")
501 | return __import__(name, *args, **kwargs)
502 |
503 | with patch("builtins.__import__", side_effect=mock_import):
504 | with patch("maverick_mcp.config.database.logger") as mock_logger:
505 | config = get_pool_config_from_settings()
506 |
507 | # Should fall back to default
508 | assert isinstance(config, DatabasePoolConfig)
509 | # Should call warning twice: import error + pool size warning
510 | assert mock_logger.warning.call_count == 2
511 | import_warning_call = mock_logger.warning.call_args_list[0]
512 | assert (
513 | "Could not import settings, using default pool configuration"
514 | in str(import_warning_call)
515 | )
516 |
517 |
518 | class TestUtilityFunctions:
519 | """Test utility functions."""
520 |
521 | def test_create_monitored_engine_kwargs(self):
522 | """Test monitored engine kwargs creation."""
523 | config = DatabasePoolConfig(
524 | pool_size=15,
525 | max_overflow=8,
526 | pool_timeout=45,
527 | pool_recycle=1800,
528 | pool_pre_ping=True,
529 | echo_pool=False,
530 | expected_concurrent_users=18, # Reduce to fit total capacity of 23
531 | connections_per_user=1.0,
532 | )
533 |
534 | database_url = "postgresql://user:pass@localhost/test"
535 | kwargs = create_monitored_engine_kwargs(database_url, config)
536 |
537 | expected = {
538 | "url": database_url,
539 | "poolclass": QueuePool,
540 | "pool_size": 15,
541 | "max_overflow": 8,
542 | "pool_timeout": 45,
543 | "pool_recycle": 1800,
544 | "pool_pre_ping": True,
545 | "echo_pool": False,
546 | "connect_args": {
547 | "application_name": "maverick_mcp",
548 | },
549 | }
550 |
551 | assert kwargs == expected
552 |
553 | @patch("sqlalchemy.create_engine")
554 | @patch("maverick_mcp.config.database.get_pool_config_from_settings")
555 | def test_create_engine_with_enhanced_config(
556 | self, mock_get_config, mock_create_engine
557 | ):
558 | """Test complete engine creation with monitoring."""
559 | mock_config = Mock(spec=DatabasePoolConfig)
560 | mock_config.pool_size = 20
561 | mock_config.max_overflow = 10
562 | mock_config.get_pool_kwargs.return_value = {"pool_size": 20}
563 | mock_config.setup_pool_monitoring = Mock()
564 | mock_get_config.return_value = mock_config
565 |
566 | mock_engine = Mock()
567 | mock_create_engine.return_value = mock_engine
568 |
569 | database_url = "postgresql://user:pass@localhost/test"
570 | result = create_engine_with_enhanced_config(database_url)
571 |
572 | # Verify engine creation and monitoring setup
573 | assert result is mock_engine
574 | mock_create_engine.assert_called_once()
575 | mock_config.setup_pool_monitoring.assert_called_once_with(mock_engine)
576 |
577 | def test_validate_production_config_valid(self):
578 | """Test production validation for valid configuration."""
579 | config = DatabasePoolConfig(
580 | pool_size=25,
581 | max_overflow=15,
582 | pool_timeout=30,
583 | pool_recycle=3600,
584 | )
585 |
586 | with patch("maverick_mcp.config.database.logger") as mock_logger:
587 | result = validate_production_config(config)
588 |
589 | assert result is True
590 | mock_logger.info.assert_called_with(
591 | "Production configuration validation passed"
592 | )
593 |
594 | def test_validate_production_config_warnings(self):
595 | """Test production validation with warnings."""
596 | config = DatabasePoolConfig(
597 | pool_size=5, # Too small
598 | max_overflow=0, # No overflow
599 | pool_timeout=30,
600 | pool_recycle=7200, # Maximum allowed (was 8000, too high)
601 | expected_concurrent_users=4, # Reduce to fit capacity of 5
602 | connections_per_user=1.0,
603 | )
604 |
605 | with patch("maverick_mcp.config.database.logger") as mock_logger:
606 | result = validate_production_config(config)
607 |
608 | assert result is True # Warnings don't fail validation
609 |
610 | # Should log multiple warnings
611 | warning_calls = list(mock_logger.warning.call_args_list)
612 | assert (
613 | len(warning_calls) == 2
614 | ) # Small pool, no overflow (recycle=7200 is max allowed, not "too long")
615 |
616 | # Check final info message mentions warnings
617 | info_call = mock_logger.info.call_args[0][0]
618 | assert "warnings" in info_call
619 |
620 | def test_validate_production_config_errors(self):
621 | """Test production validation with errors."""
622 | config = DatabasePoolConfig(
623 | pool_size=15,
624 | max_overflow=5,
625 | pool_timeout=5, # Too aggressive
626 | pool_recycle=3600,
627 | expected_concurrent_users=18, # Reduce to fit capacity of 20
628 | connections_per_user=1.0,
629 | )
630 |
631 | with pytest.raises(
632 | ValueError, match="Production configuration validation failed"
633 | ):
634 | validate_production_config(config)
635 |
636 |
637 | class TestEventListenerBehavior:
638 | """Test SQLAlchemy event listener behavior with real scenarios."""
639 |
640 | def test_connect_event_logging(self):
641 | """Test connect event logging behavior."""
642 | config = DatabasePoolConfig(
643 | pool_size=10,
644 | echo_pool=True,
645 | expected_concurrent_users=8, # Reduce expected demand to fit capacity
646 | connections_per_user=1.0,
647 | )
648 |
649 | # Mock engine and pool
650 | mock_engine = Mock()
651 | mock_pool = Mock()
652 | mock_pool.checkedout.return_value = 7 # 70% usage
653 | mock_pool.checkedin.return_value = 3
654 | mock_engine.pool = mock_pool
655 |
656 | # Mock the event registration and capture listener functions
657 | captured_listeners = {}
658 |
659 | def mock_listens_for(target, event_name):
660 | def decorator(func):
661 | captured_listeners[event_name] = func
662 | return func
663 |
664 | return decorator
665 |
666 | with patch(
667 | "maverick_mcp.config.database.event.listens_for",
668 | side_effect=mock_listens_for,
669 | ):
670 | config.setup_pool_monitoring(mock_engine)
671 |
672 | # Verify we captured the connect listener
673 | assert "connect" in captured_listeners
674 | connect_listener = captured_listeners["connect"]
675 |
676 | # Test the listener function
677 | with patch("maverick_mcp.config.database.logger") as mock_logger:
678 | connect_listener(None, None) # dbapi_connection, connection_record
679 |
680 | # Should log warning at 70% usage (above 80% threshold would be warning)
681 | # At 70%, should not trigger warning (threshold is 80%)
682 | mock_logger.warning.assert_not_called()
683 |
684 | def test_connect_event_warning_threshold(self):
685 | """Test connect event warning threshold."""
686 | config = DatabasePoolConfig(
687 | pool_size=10,
688 | echo_pool=True,
689 | expected_concurrent_users=8, # Reduce expected demand
690 | connections_per_user=1.0,
691 | )
692 |
693 | mock_engine = Mock()
694 | mock_pool = Mock()
695 | mock_pool.checkedout.return_value = 9 # 90% usage (above 80% warning threshold)
696 | mock_pool.checkedin.return_value = 1
697 | mock_engine.pool = mock_pool
698 |
699 | # Mock the event registration and capture listener functions
700 | captured_listeners = {}
701 |
702 | def mock_listens_for(target, event_name):
703 | def decorator(func):
704 | captured_listeners[event_name] = func
705 | return func
706 |
707 | return decorator
708 |
709 | with patch(
710 | "maverick_mcp.config.database.event.listens_for",
711 | side_effect=mock_listens_for,
712 | ):
713 | config.setup_pool_monitoring(mock_engine)
714 |
715 | # Verify we captured the connect listener
716 | assert "connect" in captured_listeners
717 | connect_listener = captured_listeners["connect"]
718 |
719 | # Test warning threshold
720 | with patch("maverick_mcp.config.database.logger") as mock_logger:
721 | connect_listener(None, None)
722 |
723 | # Should log warning
724 | mock_logger.warning.assert_called_once()
725 | warning_message = mock_logger.warning.call_args[0][0]
726 | assert "Pool usage approaching capacity" in warning_message
727 |
728 | def test_connect_event_critical_threshold(self):
729 | """Test connect event critical threshold."""
730 | config = DatabasePoolConfig(
731 | pool_size=10,
732 | echo_pool=True,
733 | expected_concurrent_users=8, # Reduce expected demand
734 | connections_per_user=1.0,
735 | )
736 |
737 | mock_engine = Mock()
738 | mock_pool = Mock()
739 | mock_pool.checkedout.return_value = (
740 | 10 # 100% usage (above 95% critical threshold)
741 | )
742 | mock_pool.checkedin.return_value = 0
743 | mock_engine.pool = mock_pool
744 |
745 | # Mock the event registration and capture listener functions
746 | captured_listeners = {}
747 |
748 | def mock_listens_for(target, event_name):
749 | def decorator(func):
750 | captured_listeners[event_name] = func
751 | return func
752 |
753 | return decorator
754 |
755 | with patch(
756 | "maverick_mcp.config.database.event.listens_for",
757 | side_effect=mock_listens_for,
758 | ):
759 | config.setup_pool_monitoring(mock_engine)
760 |
761 | # Verify we captured the connect listener
762 | assert "connect" in captured_listeners
763 | connect_listener = captured_listeners["connect"]
764 |
765 | # Test critical threshold
766 | with patch("maverick_mcp.config.database.logger") as mock_logger:
767 | connect_listener(None, None)
768 |
769 | # Should log both warning and error
770 | mock_logger.warning.assert_called_once()
771 | mock_logger.error.assert_called_once()
772 | error_message = mock_logger.error.call_args[0][0]
773 | assert "Pool usage critical" in error_message
774 |
775 | def test_invalidate_event_logging(self):
776 | """Test connection invalidation event logging."""
777 | config = DatabasePoolConfig(
778 | pool_size=10,
779 | echo_pool=True,
780 | expected_concurrent_users=8, # Reduce expected demand
781 | connections_per_user=1.0,
782 | )
783 | mock_engine = Mock()
784 |
785 | # Mock the event registration and capture listener functions
786 | captured_listeners = {}
787 |
788 | def mock_listens_for(target, event_name):
789 | def decorator(func):
790 | captured_listeners[event_name] = func
791 | return func
792 |
793 | return decorator
794 |
795 | with patch(
796 | "maverick_mcp.config.database.event.listens_for",
797 | side_effect=mock_listens_for,
798 | ):
799 | config.setup_pool_monitoring(mock_engine)
800 |
801 | # Verify we captured the invalidate listener
802 | assert "invalidate" in captured_listeners
803 | invalidate_listener = captured_listeners["invalidate"]
804 |
805 | # Test the listener function
806 | with patch("maverick_mcp.config.database.logger") as mock_logger:
807 | test_exception = Exception("Connection lost")
808 | invalidate_listener(None, None, test_exception)
809 |
810 | mock_logger.warning.assert_called_once()
811 | warning_message = mock_logger.warning.call_args[0][0]
812 | assert "Connection invalidated due to error" in warning_message
813 | assert "Connection lost" in warning_message
814 |
815 |
816 | class TestRealWorldScenarios:
817 | """Test realistic usage scenarios."""
818 |
819 | def test_microservice_configuration(self):
820 | """Test configuration suitable for microservice deployment."""
821 | config = DatabasePoolConfig(
822 | pool_size=8,
823 | max_overflow=4,
824 | pool_timeout=30,
825 | pool_recycle=1800,
826 | expected_concurrent_users=10,
827 | connections_per_user=1.0,
828 | max_database_connections=50,
829 | reserved_superuser_connections=2,
830 | )
831 |
832 | # Should be valid
833 | assert config.pool_size == 8
834 |
835 | # Test monitoring setup
836 | thresholds = config.get_monitoring_thresholds()
837 | assert thresholds["warning_threshold"] == 6 # 80% of 8
838 | assert thresholds["critical_threshold"] == 7 # 95% of 8
839 |
840 | def test_high_traffic_web_app_configuration(self):
841 | """Test configuration for high-traffic web application."""
842 | config = get_high_concurrency_pool_config()
843 |
844 | # Validate it's production-ready
845 | assert validate_production_config(config) is True
846 |
847 | # Should handle expected load
848 | total_capacity = config.pool_size + config.max_overflow
849 | expected_demand = config.expected_concurrent_users * config.connections_per_user
850 | assert total_capacity >= expected_demand
851 |
852 | def test_development_to_production_migration(self):
853 | """Test migrating from development to production configuration."""
854 | # Start with development config
855 | dev_config = get_development_pool_config()
856 | assert dev_config.echo_pool is True # Debug enabled
857 | assert dev_config.pool_size == 5 # Small pool
858 |
859 | # Convert to legacy for compatibility testing
860 | legacy_config = dev_config.to_legacy_config("postgresql://localhost/test")
861 | assert isinstance(legacy_config, DatabaseConfig)
862 |
863 | # Upgrade to production config
864 | prod_config = DatabasePoolConfig.from_legacy_config(
865 | legacy_config,
866 | pool_size=30, # Production sizing
867 | max_overflow=20,
868 | expected_concurrent_users=40,
869 | max_database_connections=150,
870 | echo_pool=False, # Disable debug
871 | )
872 |
873 | # Should be production-ready
874 | assert validate_production_config(prod_config) is True
875 | assert prod_config.echo_pool is False
876 | assert prod_config.pool_size == 30
877 |
878 | def test_database_upgrade_scenario(self):
879 | """Test handling database capacity upgrades."""
880 | # Original configuration for smaller database
881 | config = DatabasePoolConfig(
882 | pool_size=20,
883 | max_overflow=10,
884 | max_database_connections=100,
885 | )
886 |
887 | # Database upgraded to higher capacity
888 | config.validate_against_database_limits(200)
889 |
890 | # Configuration should be updated
891 | assert config.max_database_connections == 200
892 |
893 | # Can now safely increase pool size
894 | larger_config = DatabasePoolConfig(
895 | pool_size=40,
896 | max_overflow=20,
897 | max_database_connections=200,
898 | expected_concurrent_users=50,
899 | connections_per_user=1.2,
900 | )
901 |
902 | # Should validate successfully
903 | assert larger_config.pool_size == 40
904 |
905 | def test_connection_exhaustion_prevention(self):
906 | """Test that configuration prevents connection exhaustion."""
907 | # Configuration that would exhaust connections
908 | with pytest.raises(ValueError, match="exceeds database capacity"):
909 | DatabasePoolConfig(
910 | pool_size=45,
911 | max_overflow=35, # Total = 80
912 | max_database_connections=75, # Available = 72 (75-3)
913 | reserved_superuser_connections=3,
914 | )
915 |
916 | # Safe configuration
917 | safe_config = DatabasePoolConfig(
918 | pool_size=30,
919 | max_overflow=20, # Total = 50
920 | max_database_connections=75, # Available = 72 (75-3)
921 | reserved_superuser_connections=3,
922 | )
923 |
924 | # Should leave room for other applications and admin access
925 | total_used = safe_config.pool_size + safe_config.max_overflow
926 | available = (
927 | safe_config.max_database_connections
928 | - safe_config.reserved_superuser_connections
929 | )
930 | assert total_used < available # Should not use ALL available connections
931 |
```