This is page 4 of 8. Use http://codebase.md/tosin2013/mcp-codebase-insight?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .bumpversion.cfg
├── .codecov.yml
├── .compile-venv-py3.11
│ ├── bin
│ │ ├── activate
│ │ ├── activate.csh
│ │ ├── activate.fish
│ │ ├── Activate.ps1
│ │ ├── coverage
│ │ ├── coverage-3.11
│ │ ├── coverage3
│ │ ├── pip
│ │ ├── pip-compile
│ │ ├── pip-sync
│ │ ├── pip3
│ │ ├── pip3.11
│ │ ├── py.test
│ │ ├── pyproject-build
│ │ ├── pytest
│ │ ├── python
│ │ ├── python3
│ │ ├── python3.11
│ │ └── wheel
│ └── pyvenv.cfg
├── .env.example
├── .github
│ └── workflows
│ ├── build-verification.yml
│ ├── publish.yml
│ └── tdd-verification.yml
├── .gitignore
├── async_fixture_wrapper.py
├── CHANGELOG.md
├── CLAUDE.md
├── codebase_structure.txt
├── component_test_runner.py
├── CONTRIBUTING.md
├── core_workflows.txt
├── debug_tests.md
├── Dockerfile
├── docs
│ ├── adrs
│ │ └── 001_use_docker_for_qdrant.md
│ ├── api.md
│ ├── components
│ │ └── README.md
│ ├── cookbook.md
│ ├── development
│ │ ├── CODE_OF_CONDUCT.md
│ │ ├── CONTRIBUTING.md
│ │ └── README.md
│ ├── documentation_map.md
│ ├── documentation_summary.md
│ ├── features
│ │ ├── adr-management.md
│ │ ├── code-analysis.md
│ │ └── documentation.md
│ ├── getting-started
│ │ ├── configuration.md
│ │ ├── docker-setup.md
│ │ ├── installation.md
│ │ ├── qdrant_setup.md
│ │ └── quickstart.md
│ ├── qdrant_setup.md
│ ├── README.md
│ ├── SSE_INTEGRATION.md
│ ├── system_architecture
│ │ └── README.md
│ ├── templates
│ │ └── adr.md
│ ├── testing_guide.md
│ ├── troubleshooting
│ │ ├── common-issues.md
│ │ └── faq.md
│ ├── vector_store_best_practices.md
│ └── workflows
│ └── README.md
├── error_logs.txt
├── examples
│ └── use_with_claude.py
├── github-actions-documentation.md
├── Makefile
├── module_summaries
│ ├── backend_summary.txt
│ ├── database_summary.txt
│ └── frontend_summary.txt
├── output.txt
├── package-lock.json
├── package.json
├── PLAN.md
├── prepare_codebase.sh
├── PULL_REQUEST.md
├── pyproject.toml
├── pytest.ini
├── README.md
├── requirements-3.11.txt
├── requirements-3.11.txt.backup
├── requirements-dev.txt
├── requirements.in
├── requirements.txt
├── run_build_verification.sh
├── run_fixed_tests.sh
├── run_test_with_path_fix.sh
├── run_tests.py
├── scripts
│ ├── check_qdrant_health.sh
│ ├── compile_requirements.sh
│ ├── load_example_patterns.py
│ ├── macos_install.sh
│ ├── README.md
│ ├── setup_qdrant.sh
│ ├── start_mcp_server.sh
│ ├── store_code_relationships.py
│ ├── store_report_in_mcp.py
│ ├── validate_knowledge_base.py
│ ├── validate_poc.py
│ ├── validate_vector_store.py
│ └── verify_build.py
├── server.py
├── setup_qdrant_collection.py
├── setup.py
├── src
│ └── mcp_codebase_insight
│ ├── __init__.py
│ ├── __main__.py
│ ├── asgi.py
│ ├── core
│ │ ├── __init__.py
│ │ ├── adr.py
│ │ ├── cache.py
│ │ ├── component_status.py
│ │ ├── config.py
│ │ ├── debug.py
│ │ ├── di.py
│ │ ├── documentation.py
│ │ ├── embeddings.py
│ │ ├── errors.py
│ │ ├── health.py
│ │ ├── knowledge.py
│ │ ├── metrics.py
│ │ ├── prompts.py
│ │ ├── sse.py
│ │ ├── state.py
│ │ ├── task_tracker.py
│ │ ├── tasks.py
│ │ └── vector_store.py
│ ├── models.py
│ ├── server_test_isolation.py
│ ├── server.py
│ ├── utils
│ │ ├── __init__.py
│ │ └── logger.py
│ └── version.py
├── start-mcpserver.sh
├── summary_document.txt
├── system-architecture.md
├── system-card.yml
├── test_fix_helper.py
├── test_fixes.md
├── test_function.txt
├── test_imports.py
├── tests
│ ├── components
│ │ ├── conftest.py
│ │ ├── test_core_components.py
│ │ ├── test_embeddings.py
│ │ ├── test_knowledge_base.py
│ │ ├── test_sse_components.py
│ │ ├── test_stdio_components.py
│ │ ├── test_task_manager.py
│ │ └── test_vector_store.py
│ ├── config
│ │ └── test_config_and_env.py
│ ├── conftest.py
│ ├── integration
│ │ ├── fixed_test2.py
│ │ ├── test_api_endpoints.py
│ │ ├── test_api_endpoints.py-e
│ │ ├── test_communication_integration.py
│ │ └── test_server.py
│ ├── README.md
│ ├── README.test.md
│ ├── test_build_verifier.py
│ └── test_file_relationships.py
└── trajectories
└── tosinakinosho
├── anthropic_filemap__claude-3-sonnet-20240229__t-0.00__p-1.00__c-3.00___db62b9
│ └── db62b9
│ └── config.yaml
├── default__claude-3-5-sonnet-20240620__t-0.00__p-1.00__c-3.00___03565e
│ └── 03565e
│ ├── 03565e.traj
│ └── config.yaml
└── default__openrouter
└── anthropic
└── claude-3.5-sonnet-20240620:beta__t-0.00__p-1.00__c-3.00___03565e
└── 03565e
├── 03565e.pred
├── 03565e.traj
└── config.yaml
```
# Files
--------------------------------------------------------------------------------
/tests/integration/test_communication_integration.py:
--------------------------------------------------------------------------------
```python
1 | import asyncio
2 | import json
3 | import pytest
4 | from unittest.mock import MagicMock, AsyncMock
5 | from tests.components.test_stdio_components import MockStdinReader, MockStdoutWriter
6 |
7 | class MockSSEClient:
8 | def __init__(self):
9 | self.events = []
10 | self.connected = True
11 |
12 | async def send(self, event):
13 | if not self.connected:
14 | raise ConnectionError("Client disconnected")
15 | self.events.append(event)
16 |
17 | def disconnect(self):
18 | self.connected = False
19 |
20 | @pytest.fixture
21 | async def mock_communication_setup():
22 | """Set up mock stdio and SSE components for integration testing."""
23 | # Set up stdio mocks
24 | stdio_reader = MockStdinReader("")
25 | stdio_writer = MockStdoutWriter()
26 |
27 | # Set up SSE mock
28 | sse_client = MockSSEClient()
29 |
30 | return stdio_reader, stdio_writer, sse_client
31 |
32 | @pytest.mark.asyncio
33 | async def test_sse_stdio_interaction(mock_communication_setup):
34 | """Test interaction between SSE and STDIO communication channels."""
35 | stdio_reader, stdio_writer, sse_client = await mock_communication_setup
36 |
37 | # Step 1: Tool registration via STDIO
38 | registration_message = {
39 | "type": "register",
40 | "tool_id": "test_tool",
41 | "capabilities": ["capability1", "capability2"]
42 | }
43 |
44 | # Override reader's input with registration message
45 | stdio_reader.input_stream.write(json.dumps(registration_message) + "\n")
46 | stdio_reader.input_stream.seek(0)
47 |
48 | # Process registration
49 | line = await stdio_reader.readline()
50 | message = json.loads(line)
51 |
52 | # Send registration acknowledgment via stdio
53 | response = {
54 | "type": "registration_success",
55 | "tool_id": message["tool_id"]
56 | }
57 | await stdio_writer.write(json.dumps(response) + "\n")
58 |
59 | # Send SSE notification about new tool
60 | sse_notification = {
61 | "type": "tool_registered",
62 | "tool_id": message["tool_id"],
63 | "capabilities": message["capabilities"]
64 | }
65 | await sse_client.send(json.dumps(sse_notification))
66 |
67 | # Verify stdio response
68 | assert "registration_success" in stdio_writer.get_output()
69 |
70 | # Verify SSE notification
71 | assert len(sse_client.events) == 1
72 | assert "tool_registered" in sse_client.events[0]
73 | assert message["tool_id"] in sse_client.events[0]
74 |
75 | # Step 2: SSE event triggering STDIO message
76 | # Reset the writer to clear previous output
77 | stdio_writer = MockStdoutWriter()
78 |
79 | # Simulate an SSE event that should trigger a STDIO message
80 | sse_event = {
81 | "type": "request",
82 | "id": "sse_to_stdio_test",
83 | "method": "test_method",
84 | "params": {"param1": "value1"}
85 | }
86 |
87 | # In a real system, this would be processed by an event handler
88 | # that would then write to STDIO. Here we simulate that directly.
89 | await sse_client.send(json.dumps(sse_event))
90 |
91 | # Simulate the STDIO response that would be generated
92 | stdio_response = {
93 | "type": "response",
94 | "id": sse_event["id"],
95 | "result": {"status": "success"}
96 | }
97 | await stdio_writer.write(json.dumps(stdio_response) + "\n")
98 |
99 | # Verify the STDIO response
100 | assert "response" in stdio_writer.get_output()
101 | assert sse_event["id"] in stdio_writer.get_output()
102 |
103 | # Step 3: Bidirectional communication with state tracking
104 | # Create a simple state tracker
105 | state = {"last_message_id": None, "message_count": 0}
106 |
107 | # Send a sequence of messages in both directions
108 | for i in range(3):
109 | # STDIO to SSE
110 | stdio_message = {
111 | "type": "notification",
112 | "id": f"msg_{i}",
113 | "data": f"data_{i}"
114 | }
115 |
116 | # In a real system, this would come from STDIO input
117 | # Here we simulate by updating state directly
118 | state["last_message_id"] = stdio_message["id"]
119 | state["message_count"] += 1
120 |
121 | # Send to SSE
122 | await sse_client.send(json.dumps(stdio_message))
123 |
124 | # SSE to STDIO
125 | sse_response = {
126 | "type": "event",
127 | "id": f"response_{i}",
128 | "in_response_to": stdio_message["id"],
129 | "data": f"response_data_{i}"
130 | }
131 |
132 | # Process SSE response and update STDIO
133 | await stdio_writer.write(json.dumps(sse_response) + "\n")
134 |
135 | # Verify the communication flow
136 | assert state["message_count"] == 3
137 | assert state["last_message_id"] == "msg_2"
138 | assert len(sse_client.events) == 5 # 1 from registration + 1 from SSE event + 3 from the loop
139 |
140 | # Verify STDIO output contains all responses
141 | stdio_output = stdio_writer.get_output()
142 | for i in range(3):
143 | assert f"response_{i}" in stdio_output
144 | assert f"response_data_{i}" in stdio_output
145 |
146 | @pytest.mark.asyncio
147 | async def test_bidirectional_communication(mock_communication_setup):
148 | """Test bidirectional communication between stdio and SSE."""
149 | stdio_reader, stdio_writer, sse_client = await mock_communication_setup
150 |
151 | # Set up test message flow
152 | stdio_messages = [
153 | {"type": "request", "id": "1", "method": "test", "data": "stdio_data"},
154 | {"type": "request", "id": "2", "method": "test", "data": "more_data"}
155 | ]
156 |
157 | # Write messages to stdio
158 | for msg in stdio_messages:
159 | stdio_reader.input_stream.write(json.dumps(msg) + "\n")
160 | stdio_reader.input_stream.seek(0)
161 |
162 | # Process messages and generate SSE events
163 | while True:
164 | line = await stdio_reader.readline()
165 | if not line:
166 | break
167 |
168 | # Process stdio message
169 | message = json.loads(line)
170 |
171 | # Generate SSE event
172 | sse_event = {
173 | "type": "event",
174 | "source": "stdio",
175 | "data": message["data"]
176 | }
177 | await sse_client.send(json.dumps(sse_event))
178 |
179 | # Send response via stdio
180 | response = {
181 | "type": "response",
182 | "id": message["id"],
183 | "status": "success"
184 | }
185 | await stdio_writer.write(json.dumps(response) + "\n")
186 |
187 | # Verify all messages were processed
188 | assert len(sse_client.events) == len(stdio_messages)
189 | assert all("stdio" in event for event in sse_client.events)
190 |
191 | # Verify stdio responses
192 | output = stdio_writer.get_output()
193 | responses = [json.loads(line) for line in output.strip().split("\n")]
194 | assert len(responses) == len(stdio_messages)
195 | assert all(resp["type"] == "response" for resp in responses)
196 |
197 | @pytest.mark.asyncio
198 | async def test_error_propagation(mock_communication_setup):
199 | """Test error propagation between stdio and SSE."""
200 | stdio_reader, stdio_writer, sse_client = await mock_communication_setup
201 |
202 | # Simulate error in stdio
203 | error_message = {
204 | "type": "request",
205 | "id": "error_test",
206 | "method": "test",
207 | "data": "error_data"
208 | }
209 | stdio_reader.input_stream.write(json.dumps(error_message) + "\n")
210 | stdio_reader.input_stream.seek(0)
211 |
212 | # Process message and simulate error
213 | line = await stdio_reader.readline()
214 | message = json.loads(line)
215 |
216 | # Generate error response in stdio
217 | error_response = {
218 | "type": "error",
219 | "id": message["id"],
220 | "error": "Test error occurred"
221 | }
222 | await stdio_writer.write(json.dumps(error_response) + "\n")
223 |
224 | # Propagate error to SSE
225 | sse_error_event = {
226 | "type": "error_event",
227 | "source": "stdio",
228 | "error": "Test error occurred",
229 | "request_id": message["id"]
230 | }
231 | await sse_client.send(json.dumps(sse_error_event))
232 |
233 | # Verify error handling
234 | assert "error" in stdio_writer.get_output()
235 | assert len(sse_client.events) == 1
236 | assert "error_event" in sse_client.events[0]
237 |
238 | @pytest.mark.asyncio
239 | async def test_connection_state_handling(mock_communication_setup):
240 | """Test handling of connection state changes."""
241 | stdio_reader, stdio_writer, sse_client = await mock_communication_setup
242 |
243 | # Test normal operation
244 | test_message = {
245 | "type": "request",
246 | "id": "state_test",
247 | "method": "test"
248 | }
249 | stdio_reader.input_stream.write(json.dumps(test_message) + "\n")
250 | stdio_reader.input_stream.seek(0)
251 |
252 | # Process message while connected
253 | line = await stdio_reader.readline()
254 | message = json.loads(line)
255 | await sse_client.send(json.dumps({"type": "event", "data": "test"}))
256 |
257 | # Simulate SSE client disconnect
258 | sse_client.disconnect()
259 |
260 | # Attempt to send message after disconnect
261 | with pytest.raises(ConnectionError):
262 | await sse_client.send(json.dumps({"type": "event", "data": "test"}))
263 |
264 | # Send disconnect notification via stdio
265 | disconnect_notification = {
266 | "type": "notification",
267 | "event": "client_disconnected"
268 | }
269 | await stdio_writer.write(json.dumps(disconnect_notification) + "\n")
270 |
271 | # Verify disconnect handling
272 | assert "client_disconnected" in stdio_writer.get_output()
273 | assert not sse_client.connected
274 |
275 | @pytest.mark.asyncio
276 | async def test_race_condition_handling(mock_communication_setup):
277 | """Test handling of potential race conditions in message processing."""
278 | stdio_reader, stdio_writer, sse_client = await mock_communication_setup
279 | messages = [
280 | {"type": "request", "id": f"race_test_{i}", "sequence": i, "data": f"data_{i}"}
281 | for i in range(5)
282 | ]
283 | import random
284 | shuffled_messages = messages.copy()
285 | random.shuffle(shuffled_messages)
286 | for msg in shuffled_messages:
287 | stdio_reader.input_stream.write(json.dumps(msg) + "\n")
288 | stdio_reader.input_stream.seek(0)
289 | received_messages = {}
290 | while True:
291 | line = await stdio_reader.readline()
292 | if not line:
293 | break
294 | message = json.loads(line)
295 | received_messages[message["sequence"]] = message
296 | await sse_client.send(json.dumps({
297 | "type": "event",
298 | "sequence": message["sequence"],
299 | "data": message["data"]
300 | }))
301 | await stdio_writer.write(json.dumps({
302 | "type": "response",
303 | "id": message["id"],
304 | "sequence": message["sequence"]
305 | }) + "\n")
306 | ordered_sequences = sorted(received_messages.keys())
307 | assert ordered_sequences == list(range(5))
308 | for i, event_json in enumerate(sse_client.events):
309 | event = json.loads(event_json)
310 | assert event["sequence"] < len(messages)
311 |
312 | @pytest.mark.asyncio
313 | async def test_resource_cleanup(mock_communication_setup):
314 | """Test proper cleanup of resources after communication ends."""
315 | stdio_reader, stdio_writer, sse_client = await mock_communication_setup
316 | allocated_resources = set()
317 | async def allocate_resource(resource_id):
318 | allocated_resources.add(resource_id)
319 | async def release_resource(resource_id):
320 | allocated_resources.remove(resource_id)
321 | message = {"type": "request", "id": "resource_test", "resource": "test_resource"}
322 | stdio_reader.input_stream.write(json.dumps(message) + "\n")
323 | stdio_reader.input_stream.seek(0)
324 | line = await stdio_reader.readline()
325 | message = json.loads(line)
326 | resource_id = message["resource"]
327 | await allocate_resource(resource_id)
328 | try:
329 | await asyncio.sleep(0.1)
330 | await stdio_writer.write(json.dumps({
331 | "type": "response",
332 | "id": message["id"],
333 | "status": "success"
334 | }) + "\n")
335 | finally:
336 | await release_resource(resource_id)
337 | assert len(allocated_resources) == 0
338 |
339 | @pytest.mark.asyncio
340 | async def test_partial_message_handling(mock_communication_setup):
341 | """Test handling of partial or truncated messages."""
342 | stdio_reader, stdio_writer, sse_client = await mock_communication_setup
343 | partial_json = '{"type": "request", "id": "partial_test", "method": "test"'
344 | stdio_reader.input_stream.write(partial_json + "\n")
345 | stdio_reader.input_stream.seek(0)
346 | line = await stdio_reader.readline()
347 | try:
348 | json.loads(line)
349 | parsed = True
350 | except json.JSONDecodeError:
351 | parsed = False
352 | error_response = {
353 | "type": "error",
354 | "error": "Invalid JSON format",
355 | "code": "PARSE_ERROR"
356 | }
357 | await stdio_writer.write(json.dumps(error_response) + "\n")
358 | assert not parsed, "Parsing should have failed with partial JSON"
359 | assert "Invalid JSON format" in stdio_writer.get_output()
360 | assert "PARSE_ERROR" in stdio_writer.get_output()
```
--------------------------------------------------------------------------------
/scripts/load_example_patterns.py:
--------------------------------------------------------------------------------
```python
1 | #!/usr/bin/env python3
2 | """Load example patterns and ADRs into the knowledge base."""
3 |
4 | import asyncio
5 | import json
6 | from pathlib import Path
7 | from datetime import datetime
8 | from uuid import uuid4
9 |
10 | from mcp_codebase_insight.core.config import ServerConfig
11 | from mcp_codebase_insight.core.knowledge import KnowledgeBase, Pattern, PatternType, PatternConfidence
12 | from mcp_codebase_insight.core.vector_store import VectorStore
13 | from mcp_codebase_insight.core.embeddings import SentenceTransformerEmbedding
14 | from mcp_codebase_insight.core.adr import ADRManager, ADRStatus
15 |
16 | # Example patterns data
17 | PATTERNS = [
18 | {
19 | "name": "Factory Method",
20 | "type": "design_pattern",
21 | "description": "Define an interface for creating an object, but let subclasses decide which class to instantiate.",
22 | "content": """
23 | class Creator:
24 | def factory_method(self):
25 | pass
26 |
27 | def operation(self):
28 | product = self.factory_method()
29 | return product.operation()
30 |
31 | class ConcreteCreator(Creator):
32 | def factory_method(self):
33 | return ConcreteProduct()
34 | """,
35 | "tags": ["creational", "factory", "object-creation"],
36 | "confidence": "high"
37 | },
38 | {
39 | "name": "Repository Pattern",
40 | "type": "architecture",
41 | "description": "Mediates between the domain and data mapping layers using a collection-like interface for accessing domain objects.",
42 | "content": """
43 | class Repository:
44 | def get(self, id: str) -> Entity:
45 | pass
46 |
47 | def add(self, entity: Entity):
48 | pass
49 |
50 | def remove(self, entity: Entity):
51 | pass
52 | """,
53 | "tags": ["data-access", "persistence", "domain-driven-design"],
54 | "confidence": "high"
55 | },
56 | {
57 | "name": "Strategy Pattern",
58 | "type": "design_pattern",
59 | "description": "Define a family of algorithms, encapsulate each one, and make them interchangeable.",
60 | "content": """
61 | class Strategy:
62 | def execute(self, data):
63 | pass
64 |
65 | class ConcreteStrategyA(Strategy):
66 | def execute(self, data):
67 | return "Algorithm A"
68 |
69 | class Context:
70 | def __init__(self, strategy: Strategy):
71 | self._strategy = strategy
72 |
73 | def execute_strategy(self, data):
74 | return self._strategy.execute(data)
75 | """,
76 | "tags": ["behavioral", "algorithm", "encapsulation"],
77 | "confidence": "high"
78 | },
79 | {
80 | "name": "Error Handling Pattern",
81 | "type": "code",
82 | "description": "Common pattern for handling errors in Python using try-except with context.",
83 | "content": """
84 | def operation_with_context():
85 | try:
86 | # Setup resources
87 | resource = setup_resource()
88 | try:
89 | # Main operation
90 | result = process_resource(resource)
91 | return result
92 | except SpecificError as e:
93 | # Handle specific error
94 | handle_specific_error(e)
95 | raise
96 | finally:
97 | # Cleanup
98 | cleanup_resource(resource)
99 | except Exception as e:
100 | # Log error with context
101 | logger.error("Operation failed", exc_info=e)
102 | raise OperationError("Operation failed") from e
103 | """,
104 | "tags": ["error-handling", "python", "best-practice"],
105 | "confidence": "high"
106 | },
107 | {
108 | "name": "Circuit Breaker",
109 | "type": "architecture",
110 | "description": "Prevent system failure by failing fast and handling recovery.",
111 | "content": """
112 | class CircuitBreaker:
113 | def __init__(self, failure_threshold, reset_timeout):
114 | self.failure_count = 0
115 | self.failure_threshold = failure_threshold
116 | self.reset_timeout = reset_timeout
117 | self.last_failure_time = None
118 | self.state = "closed"
119 |
120 | async def call(self, func, *args, **kwargs):
121 | if self._should_open():
122 | self.state = "open"
123 | raise CircuitBreakerOpen()
124 |
125 | try:
126 | result = await func(*args, **kwargs)
127 | self._reset()
128 | return result
129 | except Exception as e:
130 | self._record_failure()
131 | raise
132 | """,
133 | "tags": ["resilience", "fault-tolerance", "microservices"],
134 | "confidence": "high"
135 | }
136 | ]
137 |
138 | # Example ADRs data
139 | ADRS = [
140 | {
141 | "title": "Use FastAPI for REST API Development",
142 | "context": {
143 | "problem": "We need a modern, high-performance web framework for our REST API",
144 | "constraints": [
145 | "Must support Python 3.9+",
146 | "Must support async/await",
147 | "Must have strong type validation",
148 | "Must have good documentation"
149 | ],
150 | "assumptions": [
151 | "The team has Python experience",
152 | "Performance is a priority"
153 | ]
154 | },
155 | "options": [
156 | {
157 | "title": "Use Flask",
158 | "pros": [
159 | "Simple and familiar",
160 | "Large ecosystem",
161 | "Easy to learn"
162 | ],
163 | "cons": [
164 | "No built-in async support",
165 | "No built-in validation",
166 | "Requires many extensions"
167 | ]
168 | },
169 | {
170 | "title": "Use FastAPI",
171 | "pros": [
172 | "Built-in async support",
173 | "Automatic OpenAPI documentation",
174 | "Built-in validation with Pydantic",
175 | "High performance"
176 | ],
177 | "cons": [
178 | "Newer framework with smaller ecosystem",
179 | "Steeper learning curve for some concepts"
180 | ]
181 | },
182 | {
183 | "title": "Use Django REST Framework",
184 | "pros": [
185 | "Mature and stable",
186 | "Full-featured",
187 | "Large community"
188 | ],
189 | "cons": [
190 | "Heavier weight",
191 | "Limited async support",
192 | "Slower than alternatives"
193 | ]
194 | }
195 | ],
196 | "decision": "We will use FastAPI for our REST API development due to its modern features, performance, and built-in support for async/await and validation.",
197 | "consequences": {
198 | "positive": [
199 | "Improved API performance",
200 | "Better developer experience with type hints and validation",
201 | "Automatic API documentation"
202 | ],
203 | "negative": [
204 | "Team needs to learn new concepts (dependency injection, Pydantic)",
205 | "Fewer third-party extensions compared to Flask or Django"
206 | ]
207 | }
208 | },
209 | {
210 | "title": "Vector Database for Semantic Search",
211 | "context": {
212 | "problem": "We need a database solution for storing and searching vector embeddings for semantic code search",
213 | "constraints": [
214 | "Must support efficient vector similarity search",
215 | "Must scale to handle large codebases",
216 | "Must be easy to integrate with Python"
217 | ]
218 | },
219 | "options": [
220 | {
221 | "title": "Use Qdrant",
222 | "pros": [
223 | "Purpose-built for vector search",
224 | "Good Python client",
225 | "Fast similarity search",
226 | "Support for filters"
227 | ],
228 | "cons": [
229 | "Relatively new project",
230 | "Limited community compared to alternatives"
231 | ]
232 | },
233 | {
234 | "title": "Use Elasticsearch with vector capabilities",
235 | "pros": [
236 | "Mature product",
237 | "Well-known in industry",
238 | "Many features beyond vector search"
239 | ],
240 | "cons": [
241 | "More complex to set up",
242 | "Not optimized exclusively for vector search",
243 | "Higher resource requirements"
244 | ]
245 | },
246 | {
247 | "title": "Build custom solution with NumPy/FAISS",
248 | "pros": [
249 | "Complete control over implementation",
250 | "No external service dependency",
251 | "Can optimize for specific needs"
252 | ],
253 | "cons": [
254 | "Significant development effort",
255 | "Need to handle persistence manually",
256 | "Maintenance burden"
257 | ]
258 | }
259 | ],
260 | "decision": "We will use Qdrant for vector storage and similarity search due to its performance, ease of use, and purpose-built design for vector operations.",
261 | "consequences": {
262 | "positive": [
263 | "Fast similarity search with minimal setup",
264 | "Simple API for vector operations",
265 | "Good scalability as codebase grows"
266 | ],
267 | "negative": [
268 | "New dependency to maintain",
269 | "Team needs to learn Qdrant-specific concepts"
270 | ]
271 | }
272 | }
273 | ]
274 |
275 | async def main():
276 | """Load patterns and ADRs into knowledge base."""
277 | try:
278 | # Create config
279 | config = ServerConfig()
280 |
281 | # Initialize components
282 | embedder = SentenceTransformerEmbedding(config.embedding_model)
283 | vector_store = VectorStore(
284 | url=config.qdrant_url,
285 | embedder=embedder,
286 | collection_name=config.collection_name,
287 | vector_name="fast-all-minilm-l6-v2"
288 | )
289 |
290 | # Initialize vector store
291 | await vector_store.initialize()
292 |
293 | # Create knowledge base
294 | kb = KnowledgeBase(config, vector_store)
295 | await kb.initialize()
296 |
297 | # Create patterns directory if it doesn't exist
298 | patterns_dir = Path("knowledge/patterns")
299 | patterns_dir.mkdir(parents=True, exist_ok=True)
300 |
301 | # Create ADRs directory if it doesn't exist
302 | adrs_dir = Path("docs/adrs")
303 | adrs_dir.mkdir(parents=True, exist_ok=True)
304 |
305 | # Load each pattern
306 | print("\n=== Loading Patterns ===")
307 | for pattern_data in PATTERNS:
308 | # Save pattern to knowledge base using the correct method signature
309 | created = await kb.add_pattern(
310 | name=pattern_data["name"],
311 | type=PatternType(pattern_data["type"]),
312 | description=pattern_data["description"],
313 | content=pattern_data["content"],
314 | confidence=PatternConfidence(pattern_data["confidence"]),
315 | tags=pattern_data["tags"]
316 | )
317 |
318 | print(f"Added pattern: {created.name}")
319 |
320 | # Save pattern to file
321 | pattern_file = patterns_dir / f"{created.id}.json"
322 | with open(pattern_file, "w") as f:
323 | json.dump({
324 | "id": str(created.id),
325 | "name": created.name,
326 | "type": created.type.value,
327 | "description": created.description,
328 | "content": created.content,
329 | "tags": created.tags,
330 | "confidence": created.confidence.value,
331 | "created_at": created.created_at.isoformat(),
332 | "updated_at": created.updated_at.isoformat()
333 | }, f, indent=2)
334 |
335 | print("\nAll patterns loaded successfully!")
336 |
337 | # Initialize ADR manager
338 | print("\n=== Loading ADRs ===")
339 | adr_manager = ADRManager(config)
340 | await adr_manager.initialize()
341 |
342 | # Load each ADR
343 | for adr_data in ADRS:
344 | created = await adr_manager.create_adr(
345 | title=adr_data["title"],
346 | context=adr_data["context"],
347 | options=adr_data["options"],
348 | decision=adr_data["decision"],
349 | consequences=adr_data.get("consequences")
350 | )
351 |
352 | print(f"Added ADR: {created.title}")
353 |
354 | print("\nAll ADRs loaded successfully!")
355 |
356 | # Test pattern search
357 | print("\n=== Testing Pattern Search ===")
358 | results = await kb.find_similar_patterns(
359 | "error handling in Python",
360 | limit=2
361 | )
362 |
363 | print("\nSearch results:")
364 | for result in results:
365 | print(f"- {result.pattern.name} (score: {result.similarity_score:.2f})")
366 |
367 | # Test ADR listing
368 | print("\n=== Testing ADR Listing ===")
369 | adrs = await adr_manager.list_adrs()
370 |
371 | print(f"\nFound {len(adrs)} ADRs:")
372 | for adr in adrs:
373 | print(f"- {adr.title} (status: {adr.status})")
374 |
375 | except Exception as e:
376 | print(f"Error loading examples: {e}")
377 | raise
378 |
379 | if __name__ == "__main__":
380 | asyncio.run(main())
381 |
```
--------------------------------------------------------------------------------
/tests/config/test_config_and_env.py:
--------------------------------------------------------------------------------
```python
1 | """Tests for configuration and environment handling."""
2 |
3 | import sys
4 | import os
5 |
6 | # Ensure the src directory is in the Python path
7 | sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')))
8 |
9 | import os
10 | import asyncio
11 | import shutil
12 | import pytest
13 | import pytest_asyncio
14 | from pathlib import Path
15 | from typing import Generator
16 | from unittest.mock import patch
17 | import uuid
18 |
19 | from qdrant_client import QdrantClient
20 | from qdrant_client.http.models import Distance, VectorParams
21 |
22 | from src.mcp_codebase_insight.core.config import ServerConfig
23 | from src.mcp_codebase_insight.server import CodebaseAnalysisServer
24 |
25 | @pytest.fixture(scope="session")
26 | def event_loop() -> Generator[asyncio.AbstractEventLoop, None, None]:
27 | """Create event loop for tests."""
28 | loop = asyncio.get_event_loop_policy().new_event_loop()
29 | yield loop
30 | loop.close()
31 |
32 | @pytest.fixture
33 | def env_vars(tmp_path):
34 | """Set up test environment variables and clean up test directories."""
35 | original_env = dict(os.environ)
36 | test_dirs = {
37 | "MCP_DOCS_CACHE_DIR": tmp_path / "test_docs",
38 | "MCP_ADR_DIR": tmp_path / "test_docs/adrs",
39 | "MCP_KB_STORAGE_DIR": tmp_path / "test_knowledge",
40 | "MCP_DISK_CACHE_DIR": tmp_path / "test_cache"
41 | }
42 |
43 | test_vars = {
44 | "MCP_HOST": "127.0.0.1",
45 | "MCP_PORT": "8000",
46 | "MCP_LOG_LEVEL": "DEBUG",
47 | "MCP_DEBUG": "true",
48 | "MCP_METRICS_ENABLED": "true",
49 | "MCP_CACHE_ENABLED": "true",
50 | "MCP_QDRANT_URL": "http://localhost:6333" # Use local Qdrant server
51 | }
52 | test_vars.update({k: str(v) for k, v in test_dirs.items()})
53 |
54 | os.environ.update(test_vars)
55 | yield test_vars
56 |
57 | # Clean up test directories
58 | for dir_path in test_dirs.values():
59 | if dir_path.exists():
60 | shutil.rmtree(dir_path, ignore_errors=True)
61 |
62 | # Restore original environment
63 | os.environ.clear()
64 | os.environ.update(original_env)
65 |
66 | @pytest.fixture
67 | def test_collection_name() -> str:
68 | """Generate a unique test collection name."""
69 | return f"test_collection_{uuid.uuid4().hex[:8]}"
70 |
71 | @pytest_asyncio.fixture
72 | async def qdrant_client() -> QdrantClient:
73 | """Create a Qdrant client for tests."""
74 | client = QdrantClient(url="http://localhost:6333")
75 | yield client
76 | client.close()
77 |
78 | @pytest.mark.asyncio
79 | async def test_server_config_from_env(env_vars, tmp_path, test_collection_name: str, qdrant_client: QdrantClient):
80 | """Test server configuration from environment variables."""
81 | config = ServerConfig(
82 | host=env_vars["MCP_HOST"],
83 | port=int(env_vars["MCP_PORT"]),
84 | log_level=env_vars["MCP_LOG_LEVEL"],
85 | debug_mode=env_vars["MCP_DEBUG"].lower() == "true",
86 | docs_cache_dir=Path(env_vars["MCP_DOCS_CACHE_DIR"]),
87 | adr_dir=Path(env_vars["MCP_ADR_DIR"]),
88 | kb_storage_dir=Path(env_vars["MCP_KB_STORAGE_DIR"]),
89 | disk_cache_dir=Path(env_vars["MCP_DISK_CACHE_DIR"]),
90 | qdrant_url=env_vars["MCP_QDRANT_URL"],
91 | collection_name=test_collection_name
92 | )
93 |
94 | # Create test collection
95 | try:
96 | if test_collection_name in [c.name for c in qdrant_client.get_collections().collections]:
97 | qdrant_client.delete_collection(test_collection_name)
98 |
99 | qdrant_client.create_collection(
100 | collection_name=test_collection_name,
101 | vectors_config=VectorParams(
102 | size=384, # Default size for all-MiniLM-L6-v2
103 | distance=Distance.COSINE
104 | )
105 | )
106 |
107 | server = CodebaseAnalysisServer(config)
108 | await server.initialize()
109 |
110 | assert server.config.host == env_vars["MCP_HOST"]
111 | assert server.config.port == int(env_vars["MCP_PORT"])
112 | assert server.config.log_level == env_vars["MCP_LOG_LEVEL"]
113 | assert server.config.debug_mode == (env_vars["MCP_DEBUG"].lower() == "true")
114 | assert isinstance(server.config.docs_cache_dir, Path)
115 | assert isinstance(server.config.adr_dir, Path)
116 | assert isinstance(server.config.kb_storage_dir, Path)
117 | assert isinstance(server.config.disk_cache_dir, Path)
118 | finally:
119 | await server.shutdown()
120 | if test_collection_name in [c.name for c in qdrant_client.get_collections().collections]:
121 | qdrant_client.delete_collection(test_collection_name)
122 |
123 | @pytest.mark.asyncio
124 | async def test_directory_creation(tmp_path, test_collection_name: str, qdrant_client: QdrantClient):
125 | """Test directory creation."""
126 | config = ServerConfig(
127 | host="localhost",
128 | port=8000,
129 | docs_cache_dir=tmp_path / "docs",
130 | adr_dir=tmp_path / "docs/adrs",
131 | kb_storage_dir=tmp_path / "knowledge",
132 | disk_cache_dir=tmp_path / "cache",
133 | qdrant_url="http://localhost:6333",
134 | collection_name=test_collection_name,
135 | cache_enabled=True # Explicitly enable cache for clarity
136 | )
137 |
138 | # Create test collection
139 | try:
140 | if test_collection_name in [c.name for c in qdrant_client.get_collections().collections]:
141 | qdrant_client.delete_collection(test_collection_name)
142 |
143 | qdrant_client.create_collection(
144 | collection_name=test_collection_name,
145 | vectors_config=VectorParams(
146 | size=384, # Default size for all-MiniLM-L6-v2
147 | distance=Distance.COSINE
148 | )
149 | )
150 |
151 | # Create and initialize server
152 | server = CodebaseAnalysisServer(config)
153 | await server.initialize()
154 |
155 | # Verify directories were created
156 | assert (tmp_path / "docs").exists(), "Docs directory was not created"
157 | assert (tmp_path / "docs/adrs").exists(), "ADR directory was not created"
158 | assert (tmp_path / "knowledge").exists(), "Knowledge directory was not created"
159 | assert (tmp_path / "cache").exists(), "Cache directory was not created"
160 | finally:
161 | await server.shutdown()
162 | if test_collection_name in [c.name for c in qdrant_client.get_collections().collections]:
163 | qdrant_client.delete_collection(test_collection_name)
164 |
165 | @pytest.mark.asyncio
166 | async def test_directory_creation_with_none_cache_dir(tmp_path, test_collection_name: str, qdrant_client: QdrantClient):
167 | """Test server startup with None disk_cache_dir."""
168 | config = ServerConfig(
169 | host="localhost",
170 | port=8000,
171 | docs_cache_dir=tmp_path / "docs",
172 | adr_dir=tmp_path / "docs/adrs",
173 | kb_storage_dir=tmp_path / "knowledge",
174 | disk_cache_dir=None, # Explicitly set to None
175 | qdrant_url="http://localhost:6333",
176 | collection_name=test_collection_name,
177 | cache_enabled=True # But keep cache enabled
178 | )
179 |
180 | # Create test collection
181 | try:
182 | if test_collection_name in [c.name for c in qdrant_client.get_collections().collections]:
183 | qdrant_client.delete_collection(test_collection_name)
184 |
185 | qdrant_client.create_collection(
186 | collection_name=test_collection_name,
187 | vectors_config=VectorParams(
188 | size=384, # Default size for all-MiniLM-L6-v2
189 | distance=Distance.COSINE
190 | )
191 | )
192 |
193 | # Initialize server
194 | server = CodebaseAnalysisServer(config)
195 | await server.initialize()
196 |
197 | # When disk_cache_dir is None but cache is enabled, we should default to Path("cache")
198 | assert config.disk_cache_dir == Path("cache"), "disk_cache_dir should default to 'cache'"
199 | assert Path("cache").exists(), "Default cache directory should exist"
200 | finally:
201 | await server.shutdown()
202 | if test_collection_name in [c.name for c in qdrant_client.get_collections().collections]:
203 | qdrant_client.delete_collection(test_collection_name)
204 |
205 | @pytest.mark.asyncio
206 | async def test_directory_creation_with_cache_disabled(tmp_path, test_collection_name: str, qdrant_client: QdrantClient):
207 | """Test server startup with caching disabled."""
208 | config = ServerConfig(
209 | host="localhost",
210 | port=8000,
211 | docs_cache_dir=tmp_path / "docs",
212 | adr_dir=tmp_path / "docs/adrs",
213 | kb_storage_dir=tmp_path / "knowledge",
214 | disk_cache_dir=Path(tmp_path / "cache"), # Set a path
215 | qdrant_url="http://localhost:6333",
216 | collection_name=test_collection_name,
217 | cache_enabled=False # But disable caching
218 | )
219 |
220 | # Create test collection
221 | try:
222 | if test_collection_name in [c.name for c in qdrant_client.get_collections().collections]:
223 | qdrant_client.delete_collection(test_collection_name)
224 |
225 | qdrant_client.create_collection(
226 | collection_name=test_collection_name,
227 | vectors_config=VectorParams(
228 | size=384, # Default size for all-MiniLM-L6-v2
229 | distance=Distance.COSINE
230 | )
231 | )
232 |
233 | # Server initialization should set disk_cache_dir to None when cache_enabled is False
234 | server = CodebaseAnalysisServer(config)
235 | await server.initialize()
236 |
237 | # Verify that disk_cache_dir is None when cache_enabled is False
238 | assert config.disk_cache_dir is None, "disk_cache_dir should be None when cache_enabled is False"
239 | # And that the cache directory does not exist
240 | assert not (tmp_path / "cache").exists(), "Cache directory should not exist when cache is disabled"
241 | finally:
242 | await server.shutdown()
243 | if test_collection_name in [c.name for c in qdrant_client.get_collections().collections]:
244 | qdrant_client.delete_collection(test_collection_name)
245 |
246 | @pytest.mark.asyncio
247 | async def test_directory_creation_permission_error(tmp_path, test_collection_name: str, qdrant_client: QdrantClient):
248 | """Test directory creation with permission error."""
249 | readonly_dir = tmp_path / "readonly"
250 | readonly_dir.mkdir()
251 | readonly_dir.chmod(0o444) # Read-only
252 |
253 | config = ServerConfig(
254 | host="localhost",
255 | port=8000,
256 | docs_cache_dir=readonly_dir / "docs",
257 | adr_dir=readonly_dir / "docs/adrs",
258 | kb_storage_dir=readonly_dir / "knowledge",
259 | disk_cache_dir=readonly_dir / "cache",
260 | qdrant_url="http://localhost:6333",
261 | collection_name=test_collection_name
262 | )
263 |
264 | server = None
265 | try:
266 | # Create test collection
267 | if test_collection_name in [c.name for c in qdrant_client.get_collections().collections]:
268 | qdrant_client.delete_collection(test_collection_name)
269 |
270 | qdrant_client.create_collection(
271 | collection_name=test_collection_name,
272 | vectors_config=VectorParams(
273 | size=384, # Default size for all-MiniLM-L6-v2
274 | distance=Distance.COSINE
275 | )
276 | )
277 |
278 | server = CodebaseAnalysisServer(config)
279 | with pytest.raises(RuntimeError) as exc_info:
280 | await server.initialize()
281 | assert "Permission denied" in str(exc_info.value)
282 | finally:
283 | if server:
284 | await server.shutdown()
285 | if test_collection_name in [c.name for c in qdrant_client.get_collections().collections]:
286 | qdrant_client.delete_collection(test_collection_name)
287 | # Clean up the readonly directory
288 | readonly_dir.chmod(0o777) # Restore write permissions for cleanup
289 | if readonly_dir.exists():
290 | shutil.rmtree(readonly_dir)
291 |
292 | @pytest.mark.asyncio
293 | async def test_directory_already_exists(tmp_path, test_collection_name: str, qdrant_client: QdrantClient):
294 | """Test server initialization with pre-existing directories."""
295 | # Create directories before server initialization
296 | dirs = [
297 | tmp_path / "docs",
298 | tmp_path / "docs/adrs",
299 | tmp_path / "knowledge",
300 | tmp_path / "cache"
301 | ]
302 | for dir_path in dirs:
303 | dir_path.mkdir(parents=True, exist_ok=True)
304 |
305 | config = ServerConfig(
306 | host="localhost",
307 | port=8000,
308 | docs_cache_dir=tmp_path / "docs",
309 | adr_dir=tmp_path / "docs/adrs",
310 | kb_storage_dir=tmp_path / "knowledge",
311 | disk_cache_dir=tmp_path / "cache",
312 | qdrant_url="http://localhost:6333",
313 | collection_name=test_collection_name
314 | )
315 |
316 | # Create test collection
317 | try:
318 | if test_collection_name in [c.name for c in qdrant_client.get_collections().collections]:
319 | qdrant_client.delete_collection(test_collection_name)
320 |
321 | qdrant_client.create_collection(
322 | collection_name=test_collection_name,
323 | vectors_config=VectorParams(
324 | size=384, # Default size for all-MiniLM-L6-v2
325 | distance=Distance.COSINE
326 | )
327 | )
328 |
329 | server = CodebaseAnalysisServer(config)
330 | await server.initialize()
331 |
332 | # Verify directories still exist and are accessible
333 | for dir_path in dirs:
334 | assert dir_path.exists()
335 | assert os.access(dir_path, os.R_OK | os.W_OK)
336 | finally:
337 | await server.shutdown()
338 | if test_collection_name in [c.name for c in qdrant_client.get_collections().collections]:
339 | qdrant_client.delete_collection(test_collection_name)
340 | # Clean up
341 | for dir_path in dirs:
342 | if dir_path.exists():
343 | shutil.rmtree(dir_path)
```
--------------------------------------------------------------------------------
/scripts/store_code_relationships.py:
--------------------------------------------------------------------------------
```python
1 | #!/usr/bin/env python
2 | """
3 | Store Code Component Relationships in Vector Database
4 |
5 | This script analyzes the codebase to extract relationships between components
6 | and stores them in the vector database for use in build verification.
7 | """
8 |
9 | import os
10 | import sys
11 | import json
12 | import logging
13 | import asyncio
14 | import argparse
15 | from datetime import datetime
16 | from pathlib import Path
17 | from typing import Dict, List, Any, Set, Tuple
18 | import uuid
19 |
20 | # Add the project root to the Python path
21 | sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
22 |
23 | from src.mcp_codebase_insight.core.vector_store import VectorStore
24 | from src.mcp_codebase_insight.core.embeddings import SentenceTransformerEmbedding
25 | from qdrant_client import QdrantClient
26 | from qdrant_client.http import models as rest
27 | from qdrant_client.http.models import Filter, FieldCondition, MatchValue
28 |
29 | # Configure logging
30 | logging.basicConfig(
31 | level=logging.INFO,
32 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
33 | handlers=[
34 | logging.StreamHandler(),
35 | logging.FileHandler(Path('logs/code_relationships.log'))
36 | ]
37 | )
38 | logger = logging.getLogger('code_relationships')
39 |
40 | class CodeRelationshipAnalyzer:
41 | """Code relationship analyzer for storing component relationships in vector database."""
42 |
43 | def __init__(self, config_path: str = None):
44 | """Initialize the code relationship analyzer.
45 |
46 | Args:
47 | config_path: Path to configuration file (optional)
48 | """
49 | self.config = self._load_config(config_path)
50 | self.vector_store = None
51 | self.embedder = None
52 | self.dependency_map = {}
53 | self.critical_components = set()
54 | self.source_files = []
55 |
56 | def _load_config(self, config_path: str) -> Dict[str, Any]:
57 | """Load configuration from file or environment variables.
58 |
59 | Args:
60 | config_path: Path to configuration file
61 |
62 | Returns:
63 | Configuration dictionary
64 | """
65 | config = {
66 | 'qdrant_url': os.environ.get('QDRANT_URL', 'http://localhost:6333'),
67 | 'qdrant_api_key': os.environ.get('QDRANT_API_KEY', ''),
68 | 'collection_name': os.environ.get('COLLECTION_NAME', 'mcp-codebase-insight'),
69 | 'embedding_model': os.environ.get('EMBEDDING_MODEL', 'sentence-transformers/all-MiniLM-L6-v2'),
70 | 'source_dirs': ['src'],
71 | 'exclude_dirs': ['__pycache__', '.git', '.venv', 'test_env', 'dist', 'build'],
72 | 'critical_modules': [
73 | 'mcp_codebase_insight.core.vector_store',
74 | 'mcp_codebase_insight.core.knowledge',
75 | 'mcp_codebase_insight.server'
76 | ]
77 | }
78 |
79 | # Override with config file if provided
80 | if config_path:
81 | try:
82 | with open(config_path, 'r') as f:
83 | file_config = json.load(f)
84 | config.update(file_config)
85 | except Exception as e:
86 | logger.error(f"Failed to load config from {config_path}: {e}")
87 |
88 | return config
89 |
90 | async def initialize(self):
91 | """Initialize the analyzer."""
92 | logger.info("Initializing code relationship analyzer...")
93 |
94 | # Initialize embedder
95 | logger.info("Initializing embedder...")
96 | self.embedder = SentenceTransformerEmbedding(model_name=self.config['embedding_model'])
97 | await self.embedder.initialize()
98 |
99 | # Initialize vector store
100 | logger.info(f"Connecting to vector store at {self.config['qdrant_url']}...")
101 | self.vector_store = VectorStore(
102 | url=self.config['qdrant_url'],
103 | embedder=self.embedder,
104 | collection_name=self.config['collection_name'],
105 | api_key=self.config.get('qdrant_api_key'),
106 | vector_name="default" # Specify a vector name for the collection
107 | )
108 | await self.vector_store.initialize()
109 |
110 | # Set critical components
111 | self.critical_components = set(self.config.get('critical_modules', []))
112 |
113 | logger.info("Code relationship analyzer initialized successfully")
114 |
115 | def find_source_files(self) -> List[Path]:
116 | """Find all source files to analyze.
117 |
118 | Returns:
119 | List of source file paths
120 | """
121 | logger.info("Finding source files...")
122 |
123 | source_files = []
124 | source_dirs = [Path(dir_name) for dir_name in self.config['source_dirs']]
125 | exclude_dirs = self.config['exclude_dirs']
126 |
127 | for source_dir in source_dirs:
128 | if not source_dir.exists():
129 | logger.warning(f"Source directory {source_dir} does not exist")
130 | continue
131 |
132 | for root, dirs, files in os.walk(source_dir):
133 | # Skip excluded directories
134 | dirs[:] = [d for d in dirs if d not in exclude_dirs]
135 |
136 | for file in files:
137 | if file.endswith('.py'):
138 | source_files.append(Path(root) / file)
139 |
140 | logger.info(f"Found {len(source_files)} source files")
141 | self.source_files = source_files
142 | return source_files
143 |
144 | def analyze_file_dependencies(self, file_path: Path) -> Dict[str, List[str]]:
145 | """Analyze dependencies for a single file.
146 |
147 | Args:
148 | file_path: Path to the file to analyze
149 |
150 | Returns:
151 | Dictionary mapping module name to list of dependencies
152 | """
153 | dependencies = []
154 |
155 | try:
156 | with open(file_path, 'r', encoding='utf-8') as f:
157 | content = f.read()
158 |
159 | # Extract imports
160 | lines = content.split('\n')
161 | for line in lines:
162 | line = line.strip()
163 |
164 | # Skip comments
165 | if line.startswith('#'):
166 | continue
167 |
168 | # Handle import statements
169 | if line.startswith('import ') or ' import ' in line:
170 | if line.startswith('import '):
171 | # Handle "import module" or "import module as alias"
172 | import_part = line[7:].strip()
173 | if ' as ' in import_part:
174 | import_part = import_part.split(' as ')[0].strip()
175 | dependencies.append(import_part)
176 | elif line.startswith('from ') and ' import ' in line:
177 | # Handle "from module import something"
178 | from_part = line[5:].split(' import ')[0].strip()
179 | dependencies.append(from_part)
180 |
181 | # Convert file path to module name
182 | module_name = str(file_path).replace('/', '.').replace('\\', '.').replace('.py', '')
183 | for source_dir in self.config['source_dirs']:
184 | prefix = f"{source_dir}."
185 | if module_name.startswith(prefix):
186 | module_name = module_name[len(prefix):]
187 |
188 | return {module_name: dependencies}
189 |
190 | except Exception as e:
191 | logger.error(f"Error analyzing file {file_path}: {e}")
192 | return {}
193 |
194 | def analyze_all_dependencies(self) -> Dict[str, List[str]]:
195 | """Analyze dependencies for all source files.
196 |
197 | Returns:
198 | Dictionary mapping module names to lists of dependencies
199 | """
200 | logger.info("Analyzing dependencies for all source files...")
201 |
202 | if not self.source_files:
203 | self.find_source_files()
204 |
205 | dependency_map = {}
206 |
207 | for file_path in self.source_files:
208 | file_dependencies = self.analyze_file_dependencies(file_path)
209 | dependency_map.update(file_dependencies)
210 |
211 | logger.info(f"Analyzed dependencies for {len(dependency_map)} modules")
212 | self.dependency_map = dependency_map
213 | return dependency_map
214 |
215 | def identify_critical_components(self) -> Set[str]:
216 | """Identify critical components in the codebase.
217 |
218 | Returns:
219 | Set of critical component names
220 | """
221 | logger.info("Identifying critical components...")
222 |
223 | # Start with configured critical modules
224 | critical_components = set(self.critical_components)
225 |
226 | # Add modules with many dependents
227 | if self.dependency_map:
228 | # Count how many times each module is a dependency
229 | dependent_count = {}
230 | for module, dependencies in self.dependency_map.items():
231 | for dependency in dependencies:
232 | if dependency in dependent_count:
233 | dependent_count[dependency] += 1
234 | else:
235 | dependent_count[dependency] = 1
236 |
237 | # Add modules with more than 3 dependents to critical components
238 | for module, count in dependent_count.items():
239 | if count > 3:
240 | critical_components.add(module)
241 |
242 | logger.info(f"Identified {len(critical_components)} critical components")
243 | self.critical_components = critical_components
244 | return critical_components
245 |
246 | async def store_in_vector_database(self):
247 | """Store code relationships in vector database."""
248 | try:
249 | # Store dependency map
250 | dependency_text = json.dumps({
251 | 'type': 'dependency_map',
252 | 'dependencies': self.dependency_map
253 | })
254 | dependency_vector = await self.vector_store.embedder.embed(dependency_text)
255 | dependency_data = {
256 | 'id': str(uuid.uuid4()),
257 | 'vector': dependency_vector,
258 | 'payload': {
259 | 'type': 'dependency_map',
260 | 'timestamp': datetime.now().isoformat(),
261 | 'module_count': len(self.dependency_map)
262 | }
263 | }
264 |
265 | # Store critical components
266 | critical_text = json.dumps({
267 | 'type': 'critical_components',
268 | 'components': list(self.critical_components)
269 | })
270 | critical_vector = await self.vector_store.embedder.embed(critical_text)
271 | critical_data = {
272 | 'id': str(uuid.uuid4()),
273 | 'vector': critical_vector,
274 | 'payload': {
275 | 'type': 'critical_components',
276 | 'timestamp': datetime.now().isoformat(),
277 | 'component_count': len(self.critical_components)
278 | }
279 | }
280 |
281 | # Store build verification criteria
282 | criteria_text = json.dumps({
283 | 'type': 'build_criteria',
284 | 'critical_modules': list(self.critical_components),
285 | 'min_test_coverage': 80.0,
286 | 'max_allowed_failures': 0
287 | })
288 | criteria_vector = await self.vector_store.embedder.embed(criteria_text)
289 | criteria_data = {
290 | 'id': str(uuid.uuid4()),
291 | 'vector': criteria_vector,
292 | 'payload': {
293 | 'type': 'build_criteria',
294 | 'timestamp': datetime.now().isoformat()
295 | }
296 | }
297 |
298 | # Store all data points
299 | data_points = [dependency_data, critical_data, criteria_data]
300 | self.vector_store.client.upsert(
301 | collection_name=self.vector_store.collection_name,
302 | points=[rest.PointStruct(
303 | id=data['id'],
304 | vectors={self.vector_store.vector_name: data['vector']},
305 | payload=data['payload']
306 | ) for data in data_points]
307 | )
308 |
309 | logger.info("Successfully stored code relationships in vector database")
310 |
311 | except Exception as e:
312 | logger.error(f"Error storing in vector database: {e}")
313 | raise
314 |
315 | async def analyze_and_store(self):
316 | """Analyze code relationships and store them in the vector database."""
317 | try:
318 | # Find source files
319 | self.find_source_files()
320 |
321 | # Analyze dependencies
322 | self.analyze_all_dependencies()
323 |
324 | # Identify critical components
325 | self.identify_critical_components()
326 |
327 | # Store in vector database
328 | await self.store_in_vector_database()
329 |
330 | logger.info("Analysis and storage completed successfully")
331 | return True
332 |
333 | except Exception as e:
334 | logger.error(f"Error analyzing and storing code relationships: {e}")
335 | return False
336 |
337 | async def cleanup(self):
338 | """Clean up resources."""
339 | if self.vector_store:
340 | await self.vector_store.cleanup()
341 | await self.vector_store.close()
342 |
343 | async def main():
344 | """Main function."""
345 | parser = argparse.ArgumentParser(description="Code Relationship Analyzer")
346 | parser.add_argument("--config", help="Path to configuration file")
347 | args = parser.parse_args()
348 |
349 | # Create logs directory if it doesn't exist
350 | os.makedirs("logs", exist_ok=True)
351 |
352 | analyzer = CodeRelationshipAnalyzer(args.config)
353 |
354 | try:
355 | await analyzer.initialize()
356 | success = await analyzer.analyze_and_store()
357 |
358 | if success:
359 | logger.info("Code relationship analysis completed successfully")
360 | return 0
361 | else:
362 | logger.error("Code relationship analysis failed")
363 | return 1
364 |
365 | except Exception as e:
366 | logger.error(f"Error in code relationship analysis: {e}")
367 | return 1
368 |
369 | finally:
370 | await analyzer.cleanup()
371 |
372 | if __name__ == "__main__":
373 | sys.exit(asyncio.run(main()))
```
--------------------------------------------------------------------------------
/src/mcp_codebase_insight/core/state.py:
--------------------------------------------------------------------------------
```python
1 | """Server state management."""
2 |
3 | from dataclasses import dataclass, field
4 | from typing import Dict, Optional, List, Any, Set
5 | import asyncio
6 | from contextlib import AsyncExitStack
7 | import sys
8 | import threading
9 | from datetime import datetime
10 | import logging
11 | import uuid
12 |
13 | from ..utils.logger import get_logger
14 | from .config import ServerConfig
15 | from .di import DIContainer
16 | from .task_tracker import TaskTracker
17 | from .component_status import ComponentStatus
18 |
19 | logger = get_logger(__name__)
20 |
21 | @dataclass
22 | class ComponentState:
23 | """State tracking for a server component."""
24 | status: ComponentStatus = ComponentStatus.UNINITIALIZED
25 | error: Optional[str] = None
26 | instance: Any = None
27 | last_update: datetime = field(default_factory=datetime.utcnow)
28 | retry_count: int = 0
29 | instance_id: str = field(default_factory=lambda: str(uuid.uuid4()))
30 |
31 | class ServerState:
32 | """Global server state management."""
33 |
34 | def __init__(self):
35 | """Initialize server state."""
36 | self._init_lock = asyncio.Lock()
37 | self._cleanup_lock = asyncio.Lock()
38 | self.initialized = False
39 | self.config: Optional[ServerConfig] = None
40 | self._components: Dict[str, ComponentState] = {}
41 | self._cleanup_handlers: List[asyncio.Task] = []
42 | self._task_tracker = TaskTracker()
43 | self._instance_id = str(uuid.uuid4())
44 | logger.info(f"Created ServerState instance {self._instance_id}")
45 |
46 | def register_component(self, name: str, instance: Any = None) -> None:
47 | """Register a new component."""
48 | if name not in self._components:
49 | component_state = ComponentState()
50 | if instance:
51 | component_state.instance = instance
52 | self._components[name] = component_state
53 | logger.debug(f"Registered component: {name}")
54 |
55 | def update_component_status(
56 | self,
57 | name: str,
58 | status: ComponentStatus,
59 | error: Optional[str] = None,
60 | instance: Any = None
61 | ) -> None:
62 | """Update component status."""
63 | if name not in self._components:
64 | self.register_component(name)
65 |
66 | component = self._components[name]
67 | component.status = status
68 | component.error = error
69 | component.last_update = datetime.utcnow()
70 |
71 | if instance is not None:
72 | component.instance = instance
73 |
74 | if status == ComponentStatus.FAILED:
75 | component.retry_count += 1
76 |
77 | logger.debug(
78 | f"Component {name} status updated to {status}"
79 | f"{f' (error: {error})' if error else ''}"
80 | )
81 |
82 | def get_component(self, name: str) -> Any:
83 | """Get component instance."""
84 | if name not in self._components:
85 | logger.warning(f"Component {name} not registered")
86 | return None
87 |
88 | component = self._components[name]
89 | if component.status != ComponentStatus.INITIALIZED:
90 | logger.warning(f"Component {name} not initialized (status: {component.status.value})")
91 | return None
92 |
93 | return component.instance
94 |
95 | def register_background_task(self, task: asyncio.Task) -> None:
96 | """Register a background task for tracking and cleanup."""
97 | self._task_tracker.track_task(task)
98 | logger.debug(f"Registered background task: {task.get_name()}")
99 |
100 | async def cancel_background_tasks(self) -> None:
101 | """Cancel all tracked background tasks."""
102 | await self._task_tracker.cancel_all_tasks()
103 |
104 | async def cleanup(self) -> None:
105 | """Cleanup server components."""
106 | async with self._cleanup_lock:
107 | if not self.initialized:
108 | logger.warning("Server not initialized, nothing to clean up")
109 | return
110 |
111 | logger.info(f"Beginning cleanup for instance {self._instance_id}")
112 |
113 | # First, cancel any background tasks
114 | await self.cancel_background_tasks()
115 |
116 | # Clean up components in reverse order
117 | components = list(self._components.keys())
118 | components.reverse()
119 |
120 | for component in components:
121 | self.update_component_status(component, ComponentStatus.CLEANING)
122 | try:
123 | # Component-specific cleanup logic here
124 | comp_instance = self._components[component].instance
125 | if comp_instance and hasattr(comp_instance, 'cleanup'):
126 | await comp_instance.cleanup()
127 |
128 | self.update_component_status(component, ComponentStatus.CLEANED)
129 | except Exception as e:
130 | error_msg = f"Error cleaning up {component}: {str(e)}"
131 | logger.error(error_msg, exc_info=True)
132 | self.update_component_status(
133 | component,
134 | ComponentStatus.FAILED,
135 | error_msg
136 | )
137 |
138 | # Cancel any remaining cleanup handlers
139 | for task in self._cleanup_handlers:
140 | if not task.done():
141 | task.cancel()
142 |
143 | self.initialized = False
144 | logger.info(f"Server instance {self._instance_id} cleanup completed")
145 |
146 | def get_component_status(self) -> Dict[str, Any]:
147 | """Get status of all components."""
148 | return {
149 | name: {
150 | "status": comp.status.value,
151 | "error": comp.error,
152 | "last_update": comp.last_update.isoformat(),
153 | "retry_count": comp.retry_count,
154 | "instance_id": comp.instance_id
155 | }
156 | for name, comp in self._components.items()
157 | }
158 |
159 | def register_cleanup_handler(self, task: asyncio.Task) -> None:
160 | """Register a cleanup handler task."""
161 | self._cleanup_handlers.append(task)
162 | logger.debug(f"Registered cleanup handler: {task.get_name()}")
163 |
164 | @property
165 | def instance_id(self) -> str:
166 | """Get the unique instance ID of this server state."""
167 | return self._instance_id
168 |
169 | def list_components(self) -> List[str]:
170 | """List all registered components."""
171 | return list(self._components.keys())
172 |
173 | def get_active_tasks(self) -> Set[asyncio.Task]:
174 | """Get all currently active tasks."""
175 | return self._task_tracker.get_active_tasks()
176 |
177 | def get_task_count(self) -> int:
178 | """Get the number of currently tracked tasks."""
179 | return self._task_tracker.get_task_count()
180 |
181 | async def initialize(self) -> None:
182 | """Initialize server components."""
183 | async with self._init_lock:
184 | if self.initialized:
185 | logger.warning("Server already initialized")
186 | return
187 |
188 | logger.info(f"Beginning initialization for instance {self._instance_id}")
189 |
190 | try:
191 | # Initialize components in order
192 | components = [
193 | "database",
194 | "vector_store",
195 | "task_manager",
196 | "analysis_engine",
197 | "adr_manager",
198 | "knowledge_base",
199 | "mcp_server"
200 | ]
201 |
202 | for component in components:
203 | self.update_component_status(component, ComponentStatus.INITIALIZING)
204 | try:
205 | # Component-specific initialization logic here
206 | # await self._initialize_component(component)
207 |
208 | # For now, let's just mark them as initialized
209 | # In a real implementation, you'd create and store the actual component instances
210 |
211 | # For the vector_store component, create a real instance
212 | if component == "vector_store":
213 | from .vector_store import VectorStore
214 | from .embeddings import SentenceTransformerEmbedding
215 |
216 | # If config is available, use it to configure the vector store
217 | if self.config:
218 | embedder = SentenceTransformerEmbedding(self.config.embedding_model)
219 | vector_store = VectorStore(
220 | url=self.config.qdrant_url,
221 | embedder=embedder,
222 | collection_name=self.config.collection_name
223 | )
224 | await vector_store.initialize()
225 | self.update_component_status(
226 | "vector_store",
227 | ComponentStatus.INITIALIZED,
228 | instance=vector_store
229 | )
230 |
231 | # For the adr_manager component
232 | elif component == "adr_manager":
233 | from .adr import ADRManager
234 | if self.config:
235 | adr_manager = ADRManager(self.config)
236 | await adr_manager.initialize()
237 | self.update_component_status(
238 | "adr_manager",
239 | ComponentStatus.INITIALIZED,
240 | instance=adr_manager
241 | )
242 |
243 | # For the knowledge_base component
244 | elif component == "knowledge_base":
245 | from .knowledge import KnowledgeBase
246 | if self.config:
247 | # Get vector_store if available
248 | vector_store = self.get_component("vector_store")
249 | if vector_store:
250 | kb = KnowledgeBase(self.config, vector_store)
251 | await kb.initialize()
252 | self.update_component_status(
253 | "knowledge_base",
254 | ComponentStatus.INITIALIZED,
255 | instance=kb
256 | )
257 | else:
258 | error_msg = "Vector store not initialized, cannot initialize knowledge base"
259 | logger.error(error_msg)
260 | self.update_component_status(
261 | component,
262 | ComponentStatus.FAILED,
263 | error=error_msg
264 | )
265 |
266 | # For task_manager component
267 | elif component == "task_manager":
268 | from .tasks import TaskManager
269 | if self.config:
270 | task_manager = TaskManager(self.config)
271 | await task_manager.initialize()
272 | self.update_component_status(
273 | "task_manager",
274 | ComponentStatus.INITIALIZED,
275 | instance=task_manager
276 | )
277 |
278 | # For database component (placeholder)
279 | elif component == "database":
280 | # Mock implementation for database
281 | self.update_component_status(
282 | "database",
283 | ComponentStatus.INITIALIZED,
284 | instance={"status": "mocked"}
285 | )
286 |
287 | # For analysis_engine component (placeholder)
288 | elif component == "analysis_engine":
289 | # Mock implementation for analysis engine
290 | self.update_component_status(
291 | "analysis_engine",
292 | ComponentStatus.INITIALIZED,
293 | instance={"status": "mocked"}
294 | )
295 |
296 | # For mcp_server component (placeholder)
297 | elif component == "mcp_server":
298 | # Mock implementation for mcp server
299 | self.update_component_status(
300 | "mcp_server",
301 | ComponentStatus.INITIALIZED,
302 | instance={"status": "mocked"}
303 | )
304 |
305 | except Exception as e:
306 | error_msg = f"Failed to initialize {component}: {str(e)}"
307 | logger.error(error_msg, exc_info=True)
308 | self.update_component_status(
309 | component,
310 | ComponentStatus.FAILED,
311 | error=error_msg
312 | )
313 |
314 | # Set server as initialized if all critical components are initialized
315 | critical_components = ["vector_store", "task_manager", "mcp_server"]
316 |
317 | all_critical_initialized = all(
318 | self._components.get(c) and
319 | self._components[c].status == ComponentStatus.INITIALIZED
320 | for c in critical_components
321 | )
322 |
323 | if all_critical_initialized:
324 | self.initialized = True
325 | logger.info(f"Server instance {self._instance_id} initialized successfully")
326 | else:
327 | logger.warning(
328 | f"Server instance {self._instance_id} partially initialized "
329 | f"(some critical components failed)"
330 | )
331 |
332 | except Exception as e:
333 | error_msg = f"Failed to initialize server: {str(e)}"
334 | logger.error(error_msg, exc_info=True)
335 | raise
```
--------------------------------------------------------------------------------
/.github/workflows/build-verification.yml:
--------------------------------------------------------------------------------
```yaml
1 | name: Build Verification
2 |
3 | on:
4 | push:
5 | branches: [ main ]
6 | pull_request:
7 | branches: [ main ]
8 | workflow_dispatch:
9 | inputs:
10 | config_file:
11 | description: 'Path to verification config file'
12 | required: false
13 | default: 'verification-config.json'
14 | min_coverage:
15 | description: 'Minimum test coverage percentage'
16 | required: false
17 | default: '80.0'
18 | max_failures:
19 | description: 'Maximum allowed test failures'
20 | required: false
21 | default: '0'
22 | python_version:
23 | description: 'Python version to use for verification'
24 | required: false
25 | default: '3.9'
26 |
27 | jobs:
28 | verify:
29 | runs-on: ubuntu-latest
30 | strategy:
31 | matrix:
32 | python-version: [ '3.10', '3.11', '3.12', '3.13' ]
33 | fail-fast: false # Continue testing other Python versions even if one fails
34 |
35 | name: Verify with Python ${{ matrix.python-version }}
36 | environment:
37 | name: production
38 | url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
39 |
40 | services:
41 | qdrant:
42 | image: qdrant/qdrant:v1.13.6
43 | ports:
44 | - 6333:6333
45 | - 6334:6334
46 |
47 | steps:
48 | - name: Checkout code
49 | uses: actions/checkout@v4
50 | with:
51 | fetch-depth: 0 # Fetch all history for dependencies analysis
52 |
53 | - name: Set up Python ${{ matrix.python-version }}
54 | uses: actions/[email protected]
55 | with:
56 | python-version: ${{ matrix.python-version }}
57 | cache: 'pip'
58 |
59 | - name: Wait for Qdrant and verify connection
60 | run: |
61 | echo "Waiting for Qdrant to start..."
62 | chmod +x scripts/check_qdrant_health.sh
63 | ./scripts/check_qdrant_health.sh "http://localhost:6333" 20 5
64 |
65 | - name: Setup private packages
66 | run: |
67 | # Create local-packages directory if it doesn't exist
68 | mkdir -p local-packages
69 |
70 | # If there are private packages in repositories, clone them here
71 | if [ -n "${{ secrets.PRIVATE_REPO_URL }}" ]; then
72 | echo "Setting up private package repository..."
73 |
74 | # Configure pip to use the private repository if provided
75 | mkdir -p ~/.pip
76 | echo "[global]" > ~/.pip/pip.conf
77 | echo "index-url = https://pypi.org/simple" >> ~/.pip/pip.conf
78 |
79 | # Add the private repository with token if available
80 | if [ -n "${{ secrets.PRIVATE_REPO_TOKEN }}" ]; then
81 | echo "extra-index-url = ${{ secrets.PRIVATE_REPO_URL }}:${{ secrets.PRIVATE_REPO_TOKEN }}@simple" >> ~/.pip/pip.conf
82 | else
83 | echo "extra-index-url = ${{ secrets.PRIVATE_REPO_URL }}/simple" >> ~/.pip/pip.conf
84 | fi
85 | fi
86 |
87 | # If there are local Git repositories for dependencies, clone them
88 | if [ -n "${{ secrets.MCP_SERVER_QDRANT_REPO }}" ]; then
89 | echo "Cloning mcp-server-qdrant from repository..."
90 | git clone "${{ secrets.MCP_SERVER_QDRANT_REPO }}" local-packages/mcp-server-qdrant
91 |
92 | # Install the package in development mode
93 | cd local-packages/mcp-server-qdrant
94 | pip install -e .
95 | cd ../../
96 | fi
97 |
98 | # Similarly for uvx package if needed
99 | if [ -n "${{ secrets.UVX_REPO }}" ]; then
100 | echo "Cloning uvx from repository..."
101 | git clone "${{ secrets.UVX_REPO }}" local-packages/uvx
102 |
103 | # Install the package in development mode
104 | cd local-packages/uvx
105 | pip install -e .
106 | cd ../../
107 | fi
108 |
109 | - name: Install dependencies
110 | run: |
111 | python -m pip install --upgrade pip setuptools wheel
112 |
113 | # Make the requirements script executable
114 | chmod +x scripts/compile_requirements.sh
115 |
116 | # Set environment variables for private package handling
117 | export PRIVATE_REPO_URL="${{ secrets.PRIVATE_REPO_URL }}"
118 | export PRIVATE_REPO_TOKEN="${{ secrets.PRIVATE_REPO_TOKEN }}"
119 | export LOCAL_PACKAGE_PATHS="./local-packages"
120 |
121 | # Use the compile_requirements.sh script to generate version-specific requirements
122 | echo "Using compile_requirements.sh to generate dependencies for Python ${{ matrix.python-version }}..."
123 | # Set auto-yes for cleanup to avoid interactive prompts in CI
124 | echo "y" | ./scripts/compile_requirements.sh ${{ matrix.python-version }}
125 |
126 | # Install the generated requirements
127 | if [ -f requirements-${{ matrix.python-version }}.txt ]; then
128 | echo "Installing from version-specific requirements file..."
129 | pip install -r requirements-${{ matrix.python-version }}.txt
130 | pip install -r requirements-dev.txt
131 |
132 | # Install private packages if they're in a separate file
133 | if [ -f requirements-private-${{ matrix.python-version }}.txt ]; then
134 | echo "Installing private packages..."
135 | # Try to install private packages, but continue even if it fails
136 | pip install -r requirements-private-${{ matrix.python-version }}.txt || echo "Warning: Some private packages could not be installed"
137 | fi
138 | else
139 | echo "Version-specific requirements not found, falling back to standard requirements.txt"
140 | pip install -r requirements.txt || {
141 | echo "Error installing from requirements.txt, attempting to fix compatibility issues..."
142 | grep -v "^#" requirements.txt | cut -d= -f1 | xargs pip install
143 | }
144 | fi
145 |
146 | # Install the package in development mode
147 | pip install -e .
148 |
149 | - name: Set up environment
150 | run: |
151 | # Create required directories
152 | mkdir -p logs knowledge cache
153 |
154 | {
155 | echo "QDRANT_URL=http://localhost:6333"
156 | echo "MCP_QDRANT_URL=http://localhost:6333"
157 | echo "COLLECTION_NAME=mcp-codebase-insight-${{ matrix.python-version }}"
158 | echo "MCP_COLLECTION_NAME=mcp-codebase-insight-${{ matrix.python-version }}"
159 | echo "EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2"
160 | echo "BUILD_COMMAND=make build"
161 | echo "TEST_COMMAND=make test"
162 | echo "MIN_TEST_COVERAGE=${{ github.event.inputs.min_coverage || '40.0' }}"
163 | echo "MAX_ALLOWED_FAILURES=${{ github.event.inputs.max_failures || '0' }}"
164 | echo "CRITICAL_MODULES=mcp_codebase_insight.core.vector_store,mcp_codebase_insight.core.knowledge,mcp_codebase_insight.server"
165 | echo "PYTHON_VERSION=${{ matrix.python-version }}"
166 | } >> "$GITHUB_ENV"
167 |
168 | - name: Initialize Qdrant collection
169 | run: |
170 | echo "Creating Qdrant collection for testing..."
171 | # Create a basic Python script to initialize the collection
172 | cat > init_qdrant.py << 'EOF'
173 | import os
174 | from qdrant_client import QdrantClient
175 | from qdrant_client.http import models
176 |
177 | # Connect to Qdrant
178 | client = QdrantClient(url="http://localhost:6333")
179 | collection_name = os.environ.get("COLLECTION_NAME", "mcp-codebase-insight-${{ matrix.python-version }}")
180 |
181 | # Check if collection exists
182 | collections = client.get_collections().collections
183 | collection_names = [c.name for c in collections]
184 |
185 | if collection_name in collection_names:
186 | print(f"Collection {collection_name} already exists, recreating it...")
187 | client.delete_collection(collection_name=collection_name)
188 |
189 | # Create collection with vector size 384 (for all-MiniLM-L6-v2)
190 | client.create_collection(
191 | collection_name=collection_name,
192 | vectors_config=models.VectorParams(
193 | size=384, # Dimension for all-MiniLM-L6-v2
194 | distance=models.Distance.COSINE,
195 | ),
196 | )
197 |
198 | print(f"Successfully created collection {collection_name}")
199 | EOF
200 |
201 | # Run the initialization script
202 | python init_qdrant.py
203 |
204 | # Verify the collection was created
205 | curl -s "http://localhost:6333/collections/$COLLECTION_NAME" || (echo "Failed to create Qdrant collection" && exit 1)
206 | echo "Qdrant collection initialized successfully."
207 |
208 | - name: Create configuration file
209 | if: ${{ github.event.inputs.config_file != '' }}
210 | run: |
211 | cat > ${{ github.event.inputs.config_file }} << EOF
212 | {
213 | "success_criteria": {
214 | "min_test_coverage": ${{ github.event.inputs.min_coverage || '40.0' }},
215 | "max_allowed_failures": ${{ github.event.inputs.max_failures || '0' }},
216 | "critical_modules": ["mcp_codebase_insight.core.vector_store", "mcp_codebase_insight.core.knowledge", "mcp_codebase_insight.server"],
217 | "performance_threshold_ms": 500
218 | }
219 | }
220 | EOF
221 |
222 | - name: Run build verification
223 | id: verify-build
224 | run: |
225 | # Run specific tests that are known to pass
226 | echo "Running specific tests that are known to pass..."
227 | python -m pytest \
228 | tests/components/test_core_components.py::test_adr_manager \
229 | tests/components/test_sse_components.py::test_get_starlette_app \
230 | tests/components/test_sse_components.py::test_create_sse_server \
231 | tests/components/test_sse_components.py::test_vector_search_tool \
232 | tests/components/test_sse_components.py::test_knowledge_search_tool \
233 | tests/components/test_sse_components.py::test_adr_list_tool \
234 | tests/components/test_sse_components.py::test_task_status_tool \
235 | tests/components/test_sse_components.py::test_sse_handle_connect \
236 | tests/components/test_stdio_components.py::test_stdio_registration \
237 | tests/components/test_stdio_components.py::test_stdio_message_streaming \
238 | tests/components/test_stdio_components.py::test_stdio_error_handling \
239 | tests/components/test_stdio_components.py::test_stdio_large_message \
240 | tests/components/test_knowledge_base.py \
241 | tests/integration/test_server.py::test_vector_store_search_threshold_validation \
242 | tests/integration/test_server.py::test_vector_store_search_functionality \
243 | tests/integration/test_server.py::test_vector_store_search_error_handling \
244 | tests/integration/test_server.py::test_vector_store_search_performance \
245 | tests/integration/test_api_endpoints.py::test_health_check \
246 | tests/integration/test_api_endpoints.py::test_endpoint_integration \
247 | tests/integration/test_api_endpoints.py::test_error_handling \
248 | tests/integration/test_communication_integration.py::test_sse_stdio_interaction \
249 | tests/test_file_relationships.py \
250 | -v -p pytest_asyncio --cov=src/mcp_codebase_insight --cov-report=xml:coverage.xml --cov-report=html:htmlcov
251 |
252 | TEST_EXIT_CODE=$?
253 |
254 | CONFIG_ARG=""
255 | # Use config file if it exists and is not empty
256 | if [ -n "${{ github.event.inputs.config_file }}" ] && [ -f "${{ github.event.inputs.config_file }}" ] && [ -s "${{ github.event.inputs.config_file }}" ]; then
257 | CONFIG_ARG="--config ${{ github.event.inputs.config_file }}"
258 | python -m scripts.verify_build $CONFIG_ARG --output build-verification-report.json
259 | else
260 | python -m scripts.verify_build --output build-verification-report.json
261 | fi
262 | VERIFY_EXIT_CODE=$?
263 |
264 | # Use new output syntax
265 | if [ $TEST_EXIT_CODE -ne 0 ] || [ $VERIFY_EXIT_CODE -ne 0 ]; then
266 | echo "failed=true" >> "$GITHUB_OUTPUT"
267 | fi
268 |
269 | - name: Upload verification report
270 | uses: actions/upload-artifact@v4
271 | with:
272 | name: build-verification-report
273 | path: build-verification-report.json
274 |
275 | - name: Parse verification report
276 | id: parse-report
277 | if: always()
278 | run: |
279 | if [ -f build-verification-report.json ]; then
280 | SUMMARY=$(jq -r '.build_verification_report.summary' build-verification-report.json)
281 | echo "summary=$SUMMARY" >> "$GITHUB_OUTPUT"
282 |
283 | STATUS=$(jq -r '.build_verification_report.verification_results.overall_status' build-verification-report.json)
284 | echo "status=$STATUS" >> "$GITHUB_OUTPUT"
285 |
286 | {
287 | echo "## Build Verification Report"
288 | echo "### Status: $STATUS"
289 | echo "### Summary: $SUMMARY"
290 | echo "### Test Results"
291 | TOTAL=$(jq -r '.build_verification_report.test_summary.total' build-verification-report.json)
292 | PASSED=$(jq -r '.build_verification_report.test_summary.passed' build-verification-report.json)
293 | FAILED=$(jq -r '.build_verification_report.test_summary.failed' build-verification-report.json)
294 | COVERAGE=$(jq -r '.build_verification_report.test_summary.coverage' build-verification-report.json)
295 | echo "- Total Tests: $TOTAL"
296 | echo "- Passed: $PASSED"
297 | echo "- Failed: $FAILED"
298 | echo "- Coverage: $COVERAGE%"
299 | } > report.md
300 |
301 | if jq -e '.build_verification_report.failure_analysis' build-verification-report.json > /dev/null; then
302 | {
303 | echo "### Failures Detected"
304 | jq -r '.build_verification_report.failure_analysis[] | "- " + .description' build-verification-report.json
305 | } >> report.md
306 | fi
307 |
308 | if jq -e '.build_verification_report.contextual_verification' build-verification-report.json > /dev/null; then
309 | {
310 | echo "### Contextual Analysis"
311 | jq -r '.build_verification_report.contextual_verification[] | "#### Module: " + .module + "\n- Failure: " + .failure + "\n- Dependencies: " + (.dependencies | join(", ")) + "\n\n**Potential Causes:**\n" + (.potential_causes | map("- " + .) | join("\n")) + "\n\n**Recommended Actions:**\n" + (.recommended_actions | map("- " + .) | join("\n"))' build-verification-report.json
312 | } >> report.md
313 | fi
314 | else
315 | {
316 | echo "summary=Build verification failed - no report generated" >> "$GITHUB_OUTPUT"
317 | echo "status=FAILED" >> "$GITHUB_OUTPUT"
318 | echo "## Build Verification Failed"
319 | echo "No report was generated. Check the logs for more information."
320 | } > report.md
321 | fi
322 | cat report.md
323 |
324 | - name: Create GitHub check
325 | uses: LouisBrunner/[email protected]
326 | if: always()
327 | with:
328 | token: ${{ secrets.GITHUB_TOKEN }}
329 | name: Build Verification
330 | conclusion: ${{ steps.parse-report.outputs.status == 'PASS' && 'success' || 'failure' }}
331 | output: |
332 | {
333 | "title": "Build Verification Results",
334 | "summary": "${{ steps.parse-report.outputs.summary }}",
335 | "text": "${{ steps.parse-report.outputs.report }}"
336 | }
337 |
338 | - name: Check verification status
339 | if: steps.verify-build.outputs.failed == 'true' || steps.parse-report.outputs.status != 'PASS'
340 | run: |
341 | echo "Build verification failed!"
342 | exit 1
```
--------------------------------------------------------------------------------
/src/mcp_codebase_insight/core/tasks.py:
--------------------------------------------------------------------------------
```python
1 | """Task management module."""
2 |
3 | import asyncio
4 | from datetime import datetime
5 | from enum import Enum
6 | from typing import Dict, List, Optional
7 | from uuid import UUID, uuid4
8 | import json
9 | from pathlib import Path
10 |
11 | from pydantic import BaseModel
12 |
13 | class TaskType(str, Enum):
14 | """Task type enumeration."""
15 |
16 | CODE_ANALYSIS = "code_analysis"
17 | PATTERN_EXTRACTION = "pattern_extraction"
18 | DOCUMENTATION = "documentation"
19 | DOCUMENTATION_CRAWL = "doc_crawl"
20 | DEBUG = "debug"
21 | ADR = "adr"
22 |
23 | class TaskStatus(str, Enum):
24 | """Task status enumeration."""
25 |
26 | PENDING = "pending"
27 | IN_PROGRESS = "in_progress"
28 | COMPLETED = "completed"
29 | FAILED = "failed"
30 | CANCELLED = "cancelled"
31 |
32 | class TaskPriority(str, Enum):
33 | """Task priority enumeration."""
34 |
35 | LOW = "low"
36 | MEDIUM = "medium"
37 | HIGH = "high"
38 | CRITICAL = "critical"
39 |
40 | class Task(BaseModel):
41 | """Task model."""
42 |
43 | id: UUID
44 | type: TaskType
45 | title: str
46 | description: str
47 | status: TaskStatus
48 | priority: TaskPriority
49 | context: Dict
50 | result: Optional[Dict] = None
51 | error: Optional[str] = None
52 | created_at: datetime
53 | updated_at: datetime
54 | completed_at: Optional[datetime] = None
55 | metadata: Optional[Dict[str, str]] = None
56 |
57 | class TaskManager:
58 | """Manager for asynchronous tasks."""
59 |
60 | def __init__(
61 | self,
62 | config,
63 | adr_manager=None,
64 | debug_system=None,
65 | doc_manager=None,
66 | knowledge_base=None,
67 | prompt_manager=None
68 | ):
69 | """Initialize task manager."""
70 | self.config = config
71 | self.adr_manager = adr_manager
72 | self.debug_system = debug_system
73 | self.doc_manager = doc_manager
74 | self.kb = knowledge_base
75 | self.prompt_manager = prompt_manager
76 |
77 | # Initialize tasks directory
78 | self.tasks_dir = Path(config.docs_cache_dir) / "tasks"
79 | self.tasks_dir.mkdir(parents=True, exist_ok=True)
80 |
81 | self.tasks: Dict[UUID, Task] = {}
82 | self.task_queue: asyncio.Queue = asyncio.Queue()
83 | self.running = False
84 | self._process_task_future = None
85 | self.initialized = False
86 |
87 | async def initialize(self):
88 | """Initialize task manager and start processing tasks."""
89 | if self.initialized:
90 | return
91 |
92 | try:
93 | # Create a fresh queue
94 | self.task_queue = asyncio.Queue()
95 |
96 | # Load existing tasks from disk
97 | if self.tasks_dir.exists():
98 | for task_file in self.tasks_dir.glob("*.json"):
99 | try:
100 | with open(task_file) as f:
101 | data = json.load(f)
102 | task = Task(**data)
103 | self.tasks[task.id] = task
104 | except Exception as e:
105 | print(f"Error loading task {task_file}: {e}")
106 |
107 | # Start task processing
108 | await self.start()
109 | self.initialized = True
110 | except Exception as e:
111 | print(f"Error initializing task manager: {e}")
112 | await self.cleanup()
113 | raise RuntimeError(f"Failed to initialize task manager: {str(e)}")
114 |
115 | async def cleanup(self):
116 | """Clean up task manager and stop processing tasks."""
117 | if not self.initialized:
118 | return
119 |
120 | try:
121 | # Stop task processing
122 | await self.stop()
123 |
124 | # Save any remaining tasks
125 | for task in self.tasks.values():
126 | if task.status == TaskStatus.IN_PROGRESS:
127 | task.status = TaskStatus.FAILED
128 | task.error = "Server shutdown"
129 | task.updated_at = datetime.utcnow()
130 | await self._save_task(task)
131 | except Exception as e:
132 | print(f"Error cleaning up task manager: {e}")
133 | finally:
134 | self.initialized = False
135 |
136 | async def start(self):
137 | """Start task processing."""
138 | if not self.running:
139 | self.running = True
140 | self._process_task_future = asyncio.create_task(self._process_tasks())
141 |
142 | async def stop(self):
143 | """Stop task processing."""
144 | if self.running:
145 | self.running = False
146 | if self._process_task_future:
147 | try:
148 | # Wait for the task to finish with a timeout
149 | await asyncio.wait_for(self._process_task_future, timeout=5.0)
150 | except asyncio.TimeoutError:
151 | # If it doesn't finish in time, cancel it
152 | self._process_task_future.cancel()
153 | try:
154 | await self._process_task_future
155 | except asyncio.CancelledError:
156 | pass
157 | finally:
158 | self._process_task_future = None
159 |
160 | # Create a new empty queue instead of trying to drain the old one
161 | # This avoids task_done() issues
162 | self.task_queue = asyncio.Queue()
163 |
164 | async def _save_task(self, task: Task):
165 | """Save task to disk."""
166 | task_path = self.tasks_dir / f"{task.id}.json"
167 | with open(task_path, "w") as f:
168 | json.dump(task.model_dump(), f, indent=2, default=str)
169 |
170 | async def create_task(
171 | self,
172 | type: str,
173 | title: str,
174 | description: str,
175 | context: Dict,
176 | priority: TaskPriority = TaskPriority.MEDIUM,
177 | metadata: Optional[Dict[str, str]] = None
178 | ) -> Task:
179 | """Create a new task."""
180 | now = datetime.utcnow()
181 | task = Task(
182 | id=uuid4(),
183 | type=TaskType(type),
184 | title=title,
185 | description=description,
186 | status=TaskStatus.PENDING,
187 | priority=priority,
188 | context=context,
189 | metadata=metadata,
190 | created_at=now,
191 | updated_at=now
192 | )
193 |
194 | self.tasks[task.id] = task
195 | await self._save_task(task) # Save task to disk
196 | await self.task_queue.put(task)
197 | return task
198 |
199 | async def get_task(self, task_id: str) -> Optional[Task]:
200 | """Get task by ID."""
201 | task_path = self.tasks_dir / f"{task_id}.json"
202 | if not task_path.exists():
203 | return None
204 |
205 | with open(task_path) as f:
206 | data = json.load(f)
207 | return Task(**data)
208 |
209 | async def update_task(
210 | self,
211 | task_id: str,
212 | status: Optional[str] = None,
213 | result: Optional[Dict] = None,
214 | error: Optional[str] = None
215 | ) -> Optional[Task]:
216 | """Update task status and result."""
217 | task = await self.get_task(task_id)
218 | if not task:
219 | return None
220 |
221 | if status:
222 | task.status = status
223 | if result:
224 | task.result = result
225 | if error:
226 | task.error = error
227 |
228 | task.updated_at = datetime.utcnow()
229 | if status == "completed":
230 | task.completed_at = datetime.utcnow()
231 |
232 | await self._save_task(task)
233 | return task
234 |
235 | async def cancel_task(self, task_id: UUID) -> Optional[Task]:
236 | """Cancel a pending or in-progress task."""
237 | task = self.tasks.get(task_id)
238 | if not task:
239 | return None
240 |
241 | if task.status in [TaskStatus.PENDING, TaskStatus.IN_PROGRESS]:
242 | task.status = TaskStatus.CANCELLED
243 | task.updated_at = datetime.utcnow()
244 |
245 | return task
246 |
247 | async def list_tasks(
248 | self,
249 | type: Optional[TaskType] = None,
250 | status: Optional[TaskStatus] = None,
251 | priority: Optional[TaskPriority] = None
252 | ) -> List[Task]:
253 | """List all tasks, optionally filtered."""
254 | tasks = []
255 | for task in self.tasks.values():
256 | if type and task.type != type:
257 | continue
258 | if status and task.status != status:
259 | continue
260 | if priority and task.priority != priority:
261 | continue
262 | tasks.append(task)
263 |
264 | return sorted(tasks, key=lambda x: x.created_at)
265 |
266 | async def _process_tasks(self):
267 | """Process tasks from queue."""
268 | while self.running:
269 | try:
270 | # Use get with timeout to avoid blocking forever
271 | try:
272 | task = await asyncio.wait_for(self.task_queue.get(), timeout=1.0)
273 | except asyncio.TimeoutError:
274 | continue
275 |
276 | # Update status
277 | task.status = TaskStatus.IN_PROGRESS
278 | task.updated_at = datetime.utcnow()
279 |
280 | try:
281 | # Process task based on type
282 | if task.type == TaskType.CODE_ANALYSIS:
283 | await self._process_code_analysis(task)
284 | elif task.type == TaskType.PATTERN_EXTRACTION:
285 | result = await self._extract_patterns(task)
286 | elif task.type == TaskType.DOCUMENTATION:
287 | result = await self._generate_documentation(task)
288 | elif task.type == TaskType.DOCUMENTATION_CRAWL:
289 | result = await self._crawl_documentation(task)
290 | elif task.type == TaskType.DEBUG:
291 | result = await self._debug_issue(task)
292 | elif task.type == TaskType.ADR:
293 | result = await self._process_adr(task)
294 | else:
295 | raise ValueError(f"Unknown task type: {task.type}")
296 |
297 | # Update task with result
298 | task.result = result
299 | task.status = TaskStatus.COMPLETED
300 |
301 | except Exception as e:
302 | # Update task with error
303 | task.error = str(e)
304 | task.status = TaskStatus.FAILED
305 |
306 | task.completed_at = datetime.utcnow()
307 | task.updated_at = datetime.utcnow()
308 |
309 | # Mark task as done in the queue
310 | self.task_queue.task_done()
311 |
312 | except asyncio.CancelledError:
313 | # Don't call task_done() here since we didn't get a task
314 | break
315 |
316 | except Exception as e:
317 | # Log error but continue processing
318 | print(f"Error processing task: {e}")
319 | # Don't call task_done() here since we might not have gotten a task
320 |
321 | async def _process_code_analysis(self, task: Task) -> None:
322 | """Process a code analysis task."""
323 | try:
324 | code = task.context.get("code", "")
325 | context = task.context.get("context", {})
326 |
327 | patterns = await self.app.state.knowledge.analyze_code(
328 | code=code,
329 | language=context.get("language", "python"),
330 | purpose=context.get("purpose", "")
331 | )
332 |
333 | await self._update_task(
334 | task,
335 | status=TaskStatus.COMPLETED,
336 | result={"patterns": [p.pattern.model_dump() for p in patterns]}
337 | )
338 |
339 | except Exception as e:
340 | self.logger.error(f"Failed to process code analysis task: {str(e)}")
341 | await self._update_task(
342 | task,
343 | status=TaskStatus.FAILED,
344 | error=str(e)
345 | )
346 |
347 | async def _extract_patterns(self, task: Task) -> Dict:
348 | """Extract patterns from code."""
349 | if not self.kb:
350 | raise ValueError("Knowledge base not available")
351 |
352 | code = task.context.get("code")
353 | if not code:
354 | raise ValueError("No code provided for pattern extraction")
355 |
356 | # TODO: Implement pattern extraction logic
357 | return {
358 | "patterns": []
359 | }
360 |
361 | async def _generate_documentation(self, task: Task) -> Dict:
362 | """Generate documentation."""
363 | if not self.doc_manager:
364 | raise ValueError("Documentation manager not available")
365 |
366 | content = task.context.get("content")
367 | if not content:
368 | raise ValueError("No content provided for documentation")
369 |
370 | doc = await self.doc_manager.add_document(
371 | title=task.title,
372 | content=content,
373 | type="documentation",
374 | metadata=task.metadata
375 | )
376 |
377 | return {
378 | "document_id": str(doc.id),
379 | "path": f"docs/{doc.id}.json"
380 | }
381 |
382 | async def _crawl_documentation(self, task: Task) -> Dict:
383 | """Crawl documentation from URLs."""
384 | if not self.doc_manager:
385 | raise ValueError("Documentation manager not available")
386 |
387 | urls = task.context.get("urls")
388 | source_type = task.context.get("source_type")
389 | if not urls or not source_type:
390 | raise ValueError("Missing required fields: urls, source_type")
391 |
392 | docs = await self.doc_manager.crawl_docs(
393 | urls=urls,
394 | source_type=source_type
395 | )
396 |
397 | return {
398 | "documents": [doc.model_dump() for doc in docs],
399 | "total_documents": len(docs)
400 | }
401 |
402 | async def _debug_issue(self, task: Task) -> Dict:
403 | """Debug an issue."""
404 | if not self.debug_system:
405 | raise ValueError("Debug system not available")
406 |
407 | issue = await self.debug_system.create_issue(
408 | title=task.title,
409 | type="bug",
410 | description=task.context
411 | )
412 |
413 | steps = await self.debug_system.analyze_issue(issue.id)
414 |
415 | return {
416 | "issue_id": str(issue.id),
417 | "steps": steps
418 | }
419 |
420 | async def _process_adr(self, task: Task) -> Dict:
421 | """Process ADR-related task."""
422 | if not self.adr_manager:
423 | raise ValueError("ADR manager not available")
424 |
425 | adr = await self.adr_manager.create_adr(
426 | title=task.title,
427 | context=task.context.get("context", {}),
428 | options=task.context.get("options", []),
429 | decision=task.context.get("decision", "")
430 | )
431 |
432 | return {
433 | "adr_id": str(adr.id),
434 | "path": f"docs/adrs/{adr.id}.json"
435 | }
436 |
437 | async def _process_doc_crawl(self, task: Task) -> None:
438 | """Process a document crawl task."""
439 | try:
440 | urls = task.context.get("urls", [])
441 | source_type = task.context.get("source_type", "markdown")
442 |
443 | total_documents = 0
444 | for url in urls:
445 | try:
446 | await self.doc_manager.crawl_document(url, source_type)
447 | total_documents += 1
448 | except Exception as e:
449 | print(f"Failed to crawl document {url}: {str(e)}")
450 |
451 | task.status = TaskStatus.COMPLETED
452 | task.result = {"total_documents": total_documents}
453 | task.updated_at = datetime.utcnow()
454 | task.completed_at = datetime.utcnow()
455 | await self._save_task(task)
456 |
457 | except Exception as e:
458 | print(f"Failed to process doc crawl task: {str(e)}")
459 | task.status = TaskStatus.FAILED
460 | task.error = str(e)
461 | task.updated_at = datetime.utcnow()
462 | await self._save_task(task)
463 |
```
--------------------------------------------------------------------------------
/component_test_runner.py:
--------------------------------------------------------------------------------
```python
1 | #!/usr/bin/env python
2 | """
3 | Component Test Runner
4 |
5 | A specialized runner for executing component tests with proper async fixture handling.
6 | This bypasses the standard pytest fixture mechanisms to handle async fixtures correctly
7 | in isolated execution environments.
8 | """
9 | import os
10 | import sys
11 | import uuid
12 | import asyncio
13 | import importlib
14 | from pathlib import Path
15 | import inspect
16 | import logging
17 | import re
18 | from typing import Dict, Any, List, Callable, Tuple, Optional, Set, Awaitable
19 |
20 | # Configure logging
21 | logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
22 | logger = logging.getLogger("component-test-runner")
23 |
24 | # Import the sys module to modify path
25 | import sys
26 | sys.path.insert(0, '/Users/tosinakinosho/workspaces/mcp-codebase-insight')
27 |
28 | # Import required components directly to avoid fixture resolution issues
29 | from src.mcp_codebase_insight.core.config import ServerConfig
30 | from src.mcp_codebase_insight.core.vector_store import VectorStore
31 | from src.mcp_codebase_insight.core.embeddings import SentenceTransformerEmbedding
32 | from src.mcp_codebase_insight.core.knowledge import KnowledgeBase
33 | from src.mcp_codebase_insight.core.tasks import TaskManager
34 |
35 |
36 | async def create_test_config() -> ServerConfig:
37 | """Create a server configuration for tests."""
38 | # Generate a unique collection name for this test run
39 | collection_name = f"test_collection_{uuid.uuid4().hex[:8]}"
40 |
41 | # Check if MCP_COLLECTION_NAME is set in env, use that instead if available
42 | if "MCP_COLLECTION_NAME" in os.environ:
43 | collection_name = os.environ["MCP_COLLECTION_NAME"]
44 |
45 | logger.info(f"Using test collection: {collection_name}")
46 |
47 | config = ServerConfig(
48 | host="localhost",
49 | port=8000,
50 | log_level="DEBUG",
51 | qdrant_url="http://localhost:6333",
52 | docs_cache_dir=Path(".test_cache") / "docs",
53 | adr_dir=Path(".test_cache") / "docs/adrs",
54 | kb_storage_dir=Path(".test_cache") / "knowledge",
55 | embedding_model="all-MiniLM-L6-v2",
56 | collection_name=collection_name,
57 | debug_mode=True,
58 | metrics_enabled=False,
59 | cache_enabled=True,
60 | memory_cache_size=1000,
61 | disk_cache_dir=Path(".test_cache") / "cache"
62 | )
63 | return config
64 |
65 |
66 | async def create_embedder() -> SentenceTransformerEmbedding:
67 | """Create an embedder for tests."""
68 | logger.info("Initializing the embedder...")
69 | return SentenceTransformerEmbedding()
70 |
71 |
72 | async def create_vector_store(config: ServerConfig, embedder: SentenceTransformerEmbedding) -> VectorStore:
73 | """Create a vector store for tests."""
74 | logger.info("Initializing the vector store...")
75 | store = VectorStore(config.qdrant_url, embedder)
76 | try:
77 | await store.initialize()
78 | logger.info("Vector store initialized successfully")
79 | return store
80 | except Exception as e:
81 | logger.error(f"Failed to initialize vector store: {e}")
82 | raise RuntimeError(f"Failed to initialize vector store: {e}")
83 |
84 |
85 | async def create_knowledge_base(config: ServerConfig, vector_store: VectorStore) -> KnowledgeBase:
86 | """Create a knowledge base for tests."""
87 | logger.info("Initializing the knowledge base...")
88 | kb = KnowledgeBase(config, vector_store)
89 | try:
90 | await kb.initialize()
91 | logger.info("Knowledge base initialized successfully")
92 | return kb
93 | except Exception as e:
94 | logger.error(f"Failed to initialize knowledge base: {e}")
95 | raise RuntimeError(f"Failed to initialize knowledge base: {e}")
96 |
97 |
98 | async def create_task_manager(config: ServerConfig) -> TaskManager:
99 | """Create a task manager for tests."""
100 | logger.info("Initializing the task manager...")
101 | manager = TaskManager(config)
102 | try:
103 | await manager.initialize()
104 | logger.info("Task manager initialized successfully")
105 | return manager
106 | except Exception as e:
107 | logger.error(f"Failed to initialize task manager: {e}")
108 | raise RuntimeError(f"Failed to initialize task manager: {e}")
109 |
110 |
111 | async def create_test_metadata() -> Dict[str, Any]:
112 | """Standard test metadata for consistency across tests."""
113 | return {
114 | "type": "code",
115 | "language": "python",
116 | "title": "Test Code",
117 | "description": "Test code snippet for vector store testing",
118 | "tags": ["test", "vector"]
119 | }
120 |
121 |
122 | def create_test_code() -> str:
123 | """Provide sample code for testing task-related functionality."""
124 | return """
125 | def example_function():
126 | \"\"\"This is a test function for task manager tests.\"\"\"
127 | return "Hello, world!"
128 |
129 | class TestClass:
130 | def __init__(self):
131 | self.value = 42
132 |
133 | def method(self):
134 | return self.value
135 | """
136 |
137 |
138 | async def cleanup_vector_store(vector_store: VectorStore) -> None:
139 | """Cleanup a vector store after tests."""
140 | if vector_store and hasattr(vector_store, 'cleanup'):
141 | logger.info("Cleaning up vector store...")
142 | try:
143 | await vector_store.cleanup()
144 | logger.info("Vector store cleanup completed")
145 | except Exception as e:
146 | logger.error(f"Error during vector store cleanup: {e}")
147 |
148 |
149 | async def cleanup_knowledge_base(kb: KnowledgeBase) -> None:
150 | """Cleanup a knowledge base after tests."""
151 | if kb and hasattr(kb, 'cleanup'):
152 | logger.info("Cleaning up knowledge base...")
153 | try:
154 | await kb.cleanup()
155 | logger.info("Knowledge base cleanup completed")
156 | except Exception as e:
157 | logger.error(f"Error during knowledge base cleanup: {e}")
158 |
159 |
160 | async def cleanup_task_manager(manager: TaskManager) -> None:
161 | """Cleanup a task manager after tests."""
162 | if manager and hasattr(manager, 'cleanup'):
163 | logger.info("Cleaning up task manager...")
164 | try:
165 | await manager.cleanup()
166 | logger.info("Task manager cleanup completed")
167 | except Exception as e:
168 | logger.error(f"Error cleaning up task manager: {e}")
169 |
170 |
171 | def get_module_tests(module_path: str) -> List[str]:
172 | """Get the list of tests in a module."""
173 | logger.info(f"Analyzing module: {module_path}")
174 | with open(module_path, 'r') as file:
175 | content = file.read()
176 |
177 | # Pattern to match test functions but exclude fixtures
178 | pattern = r'async\s+def\s+(test_\w+)\s*\('
179 |
180 | # Find test functions that are not fixtures (exclude lines with @pytest.fixture)
181 | lines = content.split('\n')
182 | test_functions = []
183 |
184 | for i, line in enumerate(lines):
185 | if i > 0 and '@pytest.fixture' in lines[i-1]:
186 | continue # Skip this as it's a fixture, not a test
187 |
188 | match = re.search(pattern, line)
189 | if match:
190 | test_functions.append(match.group(1))
191 |
192 | logger.info(f"Found {len(test_functions)} tests in {module_path}")
193 | return test_functions
194 |
195 | def load_test_module(module_path: str):
196 | """Load a test module with proper path handling."""
197 | # Convert file path to module path
198 | if module_path.endswith('.py'):
199 | module_path = module_path[:-3] # Remove .py extension
200 |
201 | # Convert path separators to module separators
202 | module_name = module_path.replace('/', '.').replace('\\', '.')
203 |
204 | # Ensure we use the correct Python path
205 | if not any(p == '.' for p in sys.path):
206 | sys.path.append('.')
207 |
208 | logger.info(f"Attempting to import module: {module_name}")
209 | try:
210 | return importlib.import_module(module_name)
211 | except ImportError as e:
212 | logger.error(f"Failed to import test module {module_name}: {e}")
213 | return None
214 |
215 |
216 | async def run_component_test(module_path: str, test_name: str) -> bool:
217 | """
218 | Dynamically load and run a component test with proper fixture initialization.
219 |
220 | Args:
221 | module_path: Path to the test module
222 | test_name: Name of the test function to run
223 |
224 | Returns:
225 | True if test passed, False if it failed
226 | """
227 | logger.info(f"Running test: {module_path}::{test_name}")
228 |
229 | # Import the test module
230 | test_module = load_test_module(module_path)
231 | if not test_module:
232 | return False
233 |
234 | # Get the test function
235 | if not hasattr(test_module, test_name):
236 | logger.error(f"Test function {test_name} not found in module {module_name}")
237 | return False
238 |
239 | test_func = getattr(test_module, test_name)
240 |
241 | # Determine which fixtures the test needs
242 | required_fixtures = inspect.signature(test_func).parameters
243 | logger.info(f"Test requires fixtures: {list(required_fixtures.keys())}")
244 |
245 | # Initialize the required fixtures
246 | fixture_values = {}
247 | resources_to_cleanup = []
248 |
249 | try:
250 | # Create ServerConfig first since many other fixtures depend on it
251 | if "test_config" in required_fixtures:
252 | logger.info("Setting up test_config fixture")
253 | fixture_values["test_config"] = await create_test_config()
254 |
255 | # Create embedder if needed
256 | if "embedder" in required_fixtures:
257 | logger.info("Setting up embedder fixture")
258 | fixture_values["embedder"] = await create_embedder()
259 |
260 | # Create test metadata if needed
261 | if "test_metadata" in required_fixtures:
262 | logger.info("Setting up test_metadata fixture")
263 | fixture_values["test_metadata"] = await create_test_metadata()
264 |
265 | # Create test code if needed
266 | if "test_code" in required_fixtures:
267 | logger.info("Setting up test_code fixture")
268 | fixture_values["test_code"] = create_test_code()
269 |
270 | # Create vector store if needed
271 | if "vector_store" in required_fixtures:
272 | logger.info("Setting up vector_store fixture")
273 | if "test_config" not in fixture_values:
274 | fixture_values["test_config"] = await create_test_config()
275 | if "embedder" not in fixture_values:
276 | fixture_values["embedder"] = await create_embedder()
277 |
278 | fixture_values["vector_store"] = await create_vector_store(
279 | fixture_values["test_config"],
280 | fixture_values["embedder"]
281 | )
282 | resources_to_cleanup.append(("vector_store", fixture_values["vector_store"]))
283 |
284 | # Create knowledge base if needed
285 | if "knowledge_base" in required_fixtures:
286 | logger.info("Setting up knowledge_base fixture")
287 | if "test_config" not in fixture_values:
288 | fixture_values["test_config"] = await create_test_config()
289 | if "vector_store" not in fixture_values:
290 | if "embedder" not in fixture_values:
291 | fixture_values["embedder"] = await create_embedder()
292 | fixture_values["vector_store"] = await create_vector_store(
293 | fixture_values["test_config"],
294 | fixture_values["embedder"]
295 | )
296 | resources_to_cleanup.append(("vector_store", fixture_values["vector_store"]))
297 |
298 | fixture_values["knowledge_base"] = await create_knowledge_base(
299 | fixture_values["test_config"],
300 | fixture_values["vector_store"]
301 | )
302 | resources_to_cleanup.append(("knowledge_base", fixture_values["knowledge_base"]))
303 |
304 | # Create task manager if needed
305 | if "task_manager" in required_fixtures:
306 | logger.info("Setting up task_manager fixture")
307 | if "test_config" not in fixture_values:
308 | fixture_values["test_config"] = await create_test_config()
309 |
310 | fixture_values["task_manager"] = await create_task_manager(fixture_values["test_config"])
311 | resources_to_cleanup.append(("task_manager", fixture_values["task_manager"]))
312 |
313 | # Ensure all required fixtures are initialized
314 | missing_fixtures = set(required_fixtures.keys()) - set(fixture_values.keys())
315 | if missing_fixtures:
316 | logger.error(f"Missing required fixtures: {missing_fixtures}")
317 | return False
318 |
319 | # Run the actual test
320 | logger.info(f"Executing test with fixtures: {list(fixture_values.keys())}")
321 | test_kwargs = {name: value for name, value in fixture_values.items() if name in required_fixtures}
322 |
323 | # Check if the test function is an async function
324 | if inspect.iscoroutinefunction(test_func):
325 | # For async test functions, await them
326 | logger.info(f"Running async test: {test_name}")
327 | await test_func(**test_kwargs)
328 | else:
329 | # For regular test functions, just call them
330 | logger.info(f"Running synchronous test: {test_name}")
331 | test_func(**test_kwargs)
332 |
333 | logger.info(f"Test {test_name} completed successfully")
334 | return True
335 |
336 | except Exception as e:
337 | logger.error(f"Test {test_name} failed with error: {e}")
338 | import traceback
339 | logger.error(traceback.format_exc())
340 | return False
341 |
342 | finally:
343 | # Clean up resources in reverse order (LIFO)
344 | logger.info("Cleaning up resources...")
345 | for resource_type, resource in reversed(resources_to_cleanup):
346 | try:
347 | if resource_type == "vector_store":
348 | await cleanup_vector_store(resource)
349 | elif resource_type == "knowledge_base":
350 | await cleanup_knowledge_base(resource)
351 | elif resource_type == "task_manager":
352 | await cleanup_task_manager(resource)
353 | except Exception as e:
354 | logger.error(f"Error cleaning up {resource_type}: {e}")
355 |
356 |
357 | def main():
358 | """Run a component test with proper async fixture handling."""
359 | if len(sys.argv) < 2:
360 | print("Usage: python component_test_runner.py <module_path> <test_name>")
361 | sys.exit(1)
362 |
363 | module_path = sys.argv[1]
364 |
365 | # Configure event loop policy for macOS if needed
366 | if sys.platform == 'darwin':
367 | import platform
368 | if int(platform.mac_ver()[0].split('.')[0]) >= 10:
369 | asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
370 |
371 | try:
372 | if len(sys.argv) < 3:
373 | # No specific test provided, use module discovery
374 | tests = get_module_tests(module_path)
375 | if not tests:
376 | logger.error(f"No tests found in {module_path}")
377 | sys.exit(1)
378 |
379 | # Run all tests in the module
380 | successful_tests = 0
381 | for test_name in tests:
382 | loop = asyncio.new_event_loop()
383 | asyncio.set_event_loop(loop)
384 | test_result = loop.run_until_complete(run_component_test(module_path, test_name))
385 | loop.close()
386 | if test_result:
387 | successful_tests += 1
388 |
389 | # Report test results
390 | logger.info(f"Test Results: {successful_tests}/{len(tests)} tests passed")
391 | sys.exit(0 if successful_tests == len(tests) else 1)
392 | else:
393 | # Run a specific test
394 | test_name = sys.argv[2]
395 |
396 | loop = asyncio.new_event_loop()
397 | asyncio.set_event_loop(loop)
398 | result = loop.run_until_complete(run_component_test(module_path, test_name))
399 | loop.close()
400 | sys.exit(0 if result else 1)
401 | except KeyboardInterrupt:
402 | logger.info("Test execution interrupted")
403 | sys.exit(130) # 130 is the standard exit code for SIGINT
404 | except Exception as e:
405 | logger.error(f"Unhandled exception during test execution: {e}")
406 | import traceback
407 | logger.error(traceback.format_exc())
408 | sys.exit(1)
409 |
410 |
411 | if __name__ == "__main__":
412 | main()
413 |
```
--------------------------------------------------------------------------------
/trajectories/tosinakinosho/default__openrouter/anthropic/claude-3.5-sonnet-20240620:beta__t-0.00__p-1.00__c-3.00___03565e/03565e/config.yaml:
--------------------------------------------------------------------------------
```yaml
1 | '{"env":{"deployment":{"image":"python:3.11","port":null,"docker_args":[],"startup_timeout":180.0,"pull":"missing","remove_images":false,"python_standalone_dir":"/root","platform":null,"type":"docker"},"repo":{"path":"/Users/tosinakinosho/workspaces/mcp-codebase-insight","base_commit":"HEAD","type":"local"},"post_startup_commands":[],"post_startup_command_timeout":500,"name":"main"},"agent":{"name":"main","templates":{"system_template":"SETTING:
2 | You are an autonomous programmer, and you''re working directly in the command line
3 | with a special interface.\n\nThe special interface consists of a file editor that
4 | shows you {{WINDOW}} lines of a file at a time.\nIn addition to typical bash commands,
5 | you can also use specific commands to help you navigate and edit files.\nTo call
6 | a command, you need to invoke it with a function call/tool call.\n\nPlease note
7 | that THE EDIT COMMAND REQUIRES PROPER INDENTATION.\n\nFor example, if you are looking
8 | at this file:\n\ndef fct():\n print(\"Hello world\")\n\nand you want to edit
9 | the file to read:\n\ndef fct():\n print(\"Hello\")\n print(\"world\")\n\nyou
10 | search string should be `Hello world` and your replace string should be `\"Hello\"\\n print(\"world\")`\n(note
11 | the extra spaces before the print statement!).\n\nYou could also get the same result
12 | by search for ` print(\"Hello world\")` and replace with ` print(\"Hello\")\\n print(\"world\")`.\n\nRESPONSE
13 | FORMAT:\nYour shell prompt is formatted as follows:\n(Open file: <path>)\n(Current
14 | directory: <cwd>)\nbash-$\n\nFirst, you should _always_ include a general thought
15 | about what you''re going to do next.\nThen, for every response, you must include
16 | exactly _ONE_ tool call/function call.\n\nRemember, you should always include a
17 | _SINGLE_ tool call/function call and then wait for a response from the shell before
18 | continuing with more discussion and commands. Everything you include in the DISCUSSION
19 | section will be saved for future reference.\nIf you''d like to issue two commands
20 | at once, PLEASE DO NOT DO THAT! Please instead first submit just the first tool
21 | call, and then after receiving a response you''ll be able to issue the second .\nNote
22 | that the environment does NOT support interactive session commands (e.g. python,
23 | vim), so please do not invoke them.","instance_template":"We''re currently solving
24 | the following issue within our repository. Here''s the issue text:\nISSUE:\n{{problem_statement}}\n\nINSTRUCTIONS:\nNow,
25 | you''re going to solve this issue on your own. Your terminal session has started
26 | and you''re in the repository''s root directory. You can use any bash commands or
27 | the special interface to help you. Edit all the files you need to and run any checks
28 | or tests that you want.\nRemember, YOU SHOULD ALWAYS INCLUDE EXACTLY ONE TOOL CALL/FUNCTION
29 | CALL PER RESPONSE.\nWhen you''re satisfied with all of the changes you''ve made,
30 | you can submit your changes to the code base by simply running the submit command.\nNote
31 | however that you cannot use any interactive session commands (e.g. python, vim)
32 | in this environment, but you can write scripts and run them. E.g. you can write
33 | a python script and then run it with the python command.\n\nNOTE ABOUT THE EDIT
34 | COMMAND: Indentation really matters! When editing a file, make sure to insert appropriate
35 | indentation before each line!\n\nGENERAL IMPORTANT TIPS:\n\n1. If you run a command
36 | and it doesn''t work, try running a different command. A command that did not work
37 | once will not work the second time unless you modify it!\n\n2. If you open a file
38 | and need to get to an area around a specific line that is not in the first 100 lines,
39 | say line 583, don''t just use the scroll_down command multiple times. Instead, use
40 | the goto 583 command. It''s much quicker.\n\n3. If the bug reproduction script requires
41 | inputting/reading a specific file, such as buggy-input.png, and you''d like to understand
42 | how to input that file, conduct a search in the existing repo code, to see whether
43 | someone else has already done that. Do this by running the command: find_file \"buggy-input.png\"
44 | If that doesn''t work, use the linux ''find'' command.\n\n4. Always make sure to
45 | look at the currently open file and the current working directory (which appears
46 | right after the currently open file). The currently open file might be in a different
47 | directory than the working directory! Note that some commands, such as ''create'',
48 | open files, so they might change the current open file.\n\n5. When editing files,
49 | it is easy to accidentally to write code with incorrect indentation or make other
50 | mistakes. Always check the code after you issue an edit to make sure that it reflects
51 | what you wanted to accomplish. If it didn''t, issue another command to fix it.\n\n6.
52 | When editing files, first explain the code you want to edit and why it is causing
53 | the problem. Then explain the edit you want to make and how it fixes the problem.
54 | Explain how the edit does not break existing functionality.\n\n7. Do not try to
55 | install any packages with `pip`, `conda`, or any other way. This will usually not
56 | work. If the environment is not set up correctly, try to fix the issue without executing
57 | python code or running any tests that require the package installed.\n\nSTRATEGY:\n\n1.
58 | Always start by trying to replicate the bug that the issues discusses.\n If the
59 | issue includes code for reproducing the bug, we recommend that you re-implement
60 | that in your environment, and run it to make sure you can reproduce the bug.\n Then
61 | start trying to fix it.\n\n If the bug reproduction script does not print anything
62 | when it successfully runs, we recommend adding a print(\"Script completed successfully,
63 | no errors.\") command at the end of the file,\n so that you can be sure that the
64 | script indeed ran fine all the way through.\n\n2. Locate relevant code using the
65 | find and search commands. `open` the file you want to edit.\n\n3. Use the `edit`
66 | command to perform edits.\n\n4. When you think you''ve fixed the bug, re-run the
67 | bug reproduction script to make sure that the bug has indeed been fixed.\n\n5. Create
68 | additional tests to verify the fix in a style similar to the existing reproduction
69 | script. In particular, make sure to test edge cases.\n If you find any issues,
70 | go back to the file you edited and perform further edits.\n\n(Open file: {{open_file}})\n(Current
71 | directory: {{working_dir}})\nbash-$","next_step_template":"{{observation}}\n(Open
72 | file: {{open_file}})\n(Current directory: {{working_dir}})\nbash-$","next_step_truncated_observation_template":"Observation:
73 | {{observation}}<response clipped><NOTE>Observations should not exceeded {{max_observation_length}}
74 | characters. {{elided_chars}} characters were elided. Please try a different command
75 | that produces less output or use head/tail/grep/redirect the output to a file. Do
76 | not use interactive pagers.</NOTE>","max_observation_length":100000,"next_step_no_output_template":"Your
77 | command ran successfully and did not produce any output.\n(Open file: {{open_file}})\n(Current
78 | directory: {{working_dir}})\nbash-$","strategy_template":null,"demonstration_template":"Here
79 | is a demonstration of how to correctly accomplish this task.\nIt is included to
80 | show you how to correctly use the interface.\nYou do not need to follow exactly
81 | what is done in the demonstration.\n--- DEMONSTRATION ---\n{{demonstration}}\n---
82 | END OF DEMONSTRATION ---\n","demonstrations":["/Users/tosinakinosho/workspaces/mcp-codebase-insight/SWE-agent/trajectories/demonstrations/replay__marshmallow-code__marshmallow-1867__function_calling_replace__install-1/marshmallow-code__marshmallow-1867.traj"],"put_demos_in_history":true,"shell_check_error_template":"Your
83 | bash command contained syntax errors and was NOT executed. Please fix the syntax
84 | errors and try again. This can be the result of not adhering to the syntax for multi-line
85 | commands. Here is the output of `bash -n`:\n{{bash_stdout}}\n{{bash_stderr}}","command_cancelled_timeout_template":"The
86 | command ''{{command}}'' was cancelled because it took more than {{timeout}} seconds.
87 | Please try a different command that completes more quickly."},"tools":{"filter":{"blocklist_error_template":"Operation
88 | ''{{action}}'' is not supported by this environment.","blocklist":["vim","vi","emacs","nano","nohup","gdb","less","tail
89 | -f","python -m venv","make"],"blocklist_standalone":["python","python3","ipython","bash","sh","/bin/bash","/bin/sh","nohup","vi","vim","emacs","nano","su"],"block_unless_regex":{"radare2":"\\b(?:radare2)\\b.*\\s+-c\\s+.*","r2":"\\b(?:radare2)\\b.*\\s+-c\\s+.*"}},"bundles":[{"path":"/Users/tosinakinosho/workspaces/mcp-codebase-insight/SWE-agent/tools/registry","hidden_tools":[]},{"path":"/Users/tosinakinosho/workspaces/mcp-codebase-insight/SWE-agent/tools/defaults","hidden_tools":[]},{"path":"/Users/tosinakinosho/workspaces/mcp-codebase-insight/SWE-agent/tools/search","hidden_tools":[]},{"path":"/Users/tosinakinosho/workspaces/mcp-codebase-insight/SWE-agent/tools/edit_replace","hidden_tools":[]},{"path":"/Users/tosinakinosho/workspaces/mcp-codebase-insight/SWE-agent/tools/submit","hidden_tools":[]}],"env_variables":{"WINDOW":100,"OVERLAP":2},"registry_variables":{},"submit_command":"submit","parse_function":{"error_message":"Your
90 | output was not formatted correctly. You must always include one discussion and one
91 | command as part of your response. Make sure you do not have multiple discussion/command
92 | tags.\nPlease make sure your output precisely matches the following format:\nDISCUSSION\nDiscuss
93 | here with yourself about what your planning and what you''re going to do in this
94 | step.\n\n```\ncommand(s) that you''re going to run\n```\n","type":"thought_action"},"enable_bash_tool":true,"format_error_template":"Your
95 | output was not formatted correctly. You must always include one discussion and one
96 | command as part of your response. Make sure you do not have multiple discussion/command
97 | tags.\nPlease make sure your output precisely matches the following format:\nDISCUSSION\nDiscuss
98 | here with yourself about what your planning and what you''re going to do in this
99 | step.\n\n```\ncommand(s) that you''re going to run\n```\n","command_docs":"bash:\n docstring:
100 | runs the given command directly in bash\n signature: <command>\n arguments:\n -
101 | command (string) [required]: The bash command to execute.\n\ngoto:\n docstring:
102 | moves the window to show <line_number>\n signature: goto <line_number>\n arguments:\n -
103 | line_number (integer) [required]: the line number to move the window to\n\nopen:\n docstring:
104 | opens the file at the given path in the editor. If line_number is provided, the
105 | window will be move to include that line\n signature: open \"<path>\" [<line_number>]\n arguments:\n -
106 | path (string) [required]: the path to the file to open\n - line_number (integer)
107 | [optional]: the line number to move the window to (if not provided, the window will
108 | start at the top of the file)\n\ncreate:\n docstring: creates and opens a new file
109 | with the given name\n signature: create <filename>\n arguments:\n - filename
110 | (string) [required]: the name of the file to create\n\nscroll_up:\n docstring:
111 | moves the window up 100 lines\n signature: scroll_up\n\nscroll_down:\n docstring:
112 | moves the window down 100 lines\n signature: scroll_down\n\nfind_file:\n docstring:
113 | finds all files with the given name or pattern in dir. If dir is not provided, searches
114 | in the current directory\n signature: find_file <file_name> [<dir>]\n arguments:\n -
115 | file_name (string) [required]: the name of the file or pattern to search for. supports
116 | shell-style wildcards (e.g. *.py)\n - dir (string) [optional]: the directory
117 | to search in (if not provided, searches in the current directory)\n\nsearch_dir:\n docstring:
118 | searches for search_term in all files in dir. If dir is not provided, searches in
119 | the current directory\n signature: search_dir <search_term> [<dir>]\n arguments:\n -
120 | search_term (string) [required]: the term to search for\n - dir (string) [optional]:
121 | the directory to search in (if not provided, searches in the current directory)\n\nsearch_file:\n docstring:
122 | searches for search_term in file. If file is not provided, searches in the current
123 | open file\n signature: search_file <search_term> [<file>]\n arguments:\n -
124 | search_term (string) [required]: the term to search for\n - file (string) [optional]:
125 | the file to search in (if not provided, searches in the current open file)\n\nedit:\n docstring:
126 | Replace first occurrence of <search> with <replace> in the currently displayed lines.
127 | If replace-all is True , replace all occurrences of <search> with <replace>.\nFor
128 | example, if you are looking at this file:\ndef fct():\n print(\"Hello world\")\n\nand
129 | you want to edit the file to read:\ndef fct():\n print(\"Hello\")\n print(\"world\")\n\nyou
130 | can search for `Hello world` and replace with `\"Hello\"\\n print(\"world\")`
131 | (note the extra spaces before the print statement!).\nTips:\n1. Always include proper
132 | whitespace/indentation 2. When you are adding an if/with/try statement, you need
133 | to INDENT the block that follows, so make sure to include it in both your search
134 | and replace strings! 3. If you are wrapping code in a try statement, make sure to
135 | also add an ''except'' or ''finally'' block.\nBefore every edit, please\n1. Explain
136 | the code you want to edit and why it is causing the problem 2. Explain the edit
137 | you want to make and how it fixes the problem 3. Explain how the edit does not break
138 | existing functionality\n\n signature: edit <search> <replace> [<replace-all>]\n\n arguments:\n -
139 | search (string) [required]: the text to search for (make sure to include proper
140 | whitespace if needed)\n - replace (string) [required]: the text to replace the
141 | search with (make sure to include proper whitespace if needed)\n - replace-all
142 | (boolean) [optional]: replace all occurrences rather than the first occurrence within
143 | the displayed lines\n\ninsert:\n docstring: Insert <text> at the end of the currently
144 | opened file or after <line> if specified.\n\n signature: insert <text> [<line>]\n\n arguments:\n -
145 | text (string) [required]: the text to insert\n - line (integer) [optional]: the
146 | line number to insert the text as new lines after\n\nsubmit:\n docstring: submits
147 | the current file\n signature: submit\n\n","multi_line_command_endings":{},"submit_command_end_name":null,"reset_commands":[],"execution_timeout":30,"install_timeout":300,"total_execution_timeout":1800,"max_consecutive_execution_timeouts":3},"history_processors":[{"n":5,"polling":1,"always_remove_output_for_tags":["remove_output"],"always_keep_output_for_tags":["keep_output"],"type":"last_n_observations"}],"model":{"name":"openrouter/anthropic/claude-3.5-sonnet-20240620:beta","per_instance_cost_limit":3.0,"total_cost_limit":0.0,"per_instance_call_limit":0,"temperature":0.0,"top_p":1.0,"api_base":null,"api_version":null,"api_key":null,"stop":[],"completion_kwargs":{},"convert_system_to_user":false,"retry":{"retries":20,"min_wait":10.0,"max_wait":120.0},"delay":0.0,"fallbacks":[],"choose_api_key_by_thread":true,"max_input_tokens":null,"max_output_tokens":null},"max_requeries":3,"action_sampler":null,"type":"default"},"problem_statement":{"text":"#
148 | Debug MCP Codebase Insight Tests","extra_fields":{},"type":"text","id":"03565e"},"output_dir":"/Users/tosinakinosho/workspaces/mcp-codebase-insight/trajectories/tosinakinosho/default__openrouter/anthropic/claude-3.5-sonnet-20240620:beta__t-0.00__p-1.00__c-3.00___03565e","actions":{"open_pr":false,"pr_config":{"skip_if_commits_reference_issue":true},"apply_patch_locally":false},"env_var_path":null}'
149 |
```
--------------------------------------------------------------------------------
/trajectories/tosinakinosho/default__claude-3-5-sonnet-20240620__t-0.00__p-1.00__c-3.00___03565e/03565e/config.yaml:
--------------------------------------------------------------------------------
```yaml
1 | '{"env":{"deployment":{"image":"python:3.11","port":null,"docker_args":[],"startup_timeout":180.0,"pull":"missing","remove_images":false,"python_standalone_dir":"/root","platform":null,"type":"docker"},"repo":null,"post_startup_commands":[],"post_startup_command_timeout":500,"name":"main"},"agent":{"name":"main","templates":{"system_template":"SETTING:
2 | You are an autonomous programmer, and you''re working directly in the command line
3 | with a special interface.\n\nThe special interface consists of a file editor that
4 | shows you {{WINDOW}} lines of a file at a time.\nIn addition to typical bash commands,
5 | you can also use specific commands to help you navigate and edit files.\nTo call
6 | a command, you need to invoke it with a function call/tool call.\n\nPlease note
7 | that THE EDIT COMMAND REQUIRES PROPER INDENTATION.\n\nFor example, if you are looking
8 | at this file:\n\ndef fct():\n print(\"Hello world\")\n\nand you want to edit
9 | the file to read:\n\ndef fct():\n print(\"Hello\")\n print(\"world\")\n\nyou
10 | search string should be `Hello world` and your replace string should be `\"Hello\"\\n print(\"world\")`\n(note
11 | the extra spaces before the print statement!).\n\nYou could also get the same result
12 | by search for ` print(\"Hello world\")` and replace with ` print(\"Hello\")\\n print(\"world\")`.\n\nRESPONSE
13 | FORMAT:\nYour shell prompt is formatted as follows:\n(Open file: <path>)\n(Current
14 | directory: <cwd>)\nbash-$\n\nFirst, you should _always_ include a general thought
15 | about what you''re going to do next.\nThen, for every response, you must include
16 | exactly _ONE_ tool call/function call.\n\nRemember, you should always include a
17 | _SINGLE_ tool call/function call and then wait for a response from the shell before
18 | continuing with more discussion and commands. Everything you include in the DISCUSSION
19 | section will be saved for future reference.\nIf you''d like to issue two commands
20 | at once, PLEASE DO NOT DO THAT! Please instead first submit just the first tool
21 | call, and then after receiving a response you''ll be able to issue the second .\nNote
22 | that the environment does NOT support interactive session commands (e.g. python,
23 | vim), so please do not invoke them.","instance_template":"We''re currently solving
24 | the following issue within our repository. Here''s the issue text:\nISSUE:\n{{problem_statement}}\n\nINSTRUCTIONS:\nNow,
25 | you''re going to solve this issue on your own. Your terminal session has started
26 | and you''re in the repository''s root directory. You can use any bash commands or
27 | the special interface to help you. Edit all the files you need to and run any checks
28 | or tests that you want.\nRemember, YOU SHOULD ALWAYS INCLUDE EXACTLY ONE TOOL CALL/FUNCTION
29 | CALL PER RESPONSE.\nWhen you''re satisfied with all of the changes you''ve made,
30 | you can submit your changes to the code base by simply running the submit command.\nNote
31 | however that you cannot use any interactive session commands (e.g. python, vim)
32 | in this environment, but you can write scripts and run them. E.g. you can write
33 | a python script and then run it with the python command.\n\nNOTE ABOUT THE EDIT
34 | COMMAND: Indentation really matters! When editing a file, make sure to insert appropriate
35 | indentation before each line!\n\nGENERAL IMPORTANT TIPS:\n\n1. If you run a command
36 | and it doesn''t work, try running a different command. A command that did not work
37 | once will not work the second time unless you modify it!\n\n2. If you open a file
38 | and need to get to an area around a specific line that is not in the first 100 lines,
39 | say line 583, don''t just use the scroll_down command multiple times. Instead, use
40 | the goto 583 command. It''s much quicker.\n\n3. If the bug reproduction script requires
41 | inputting/reading a specific file, such as buggy-input.png, and you''d like to understand
42 | how to input that file, conduct a search in the existing repo code, to see whether
43 | someone else has already done that. Do this by running the command: find_file \"buggy-input.png\"
44 | If that doesn''t work, use the linux ''find'' command.\n\n4. Always make sure to
45 | look at the currently open file and the current working directory (which appears
46 | right after the currently open file). The currently open file might be in a different
47 | directory than the working directory! Note that some commands, such as ''create'',
48 | open files, so they might change the current open file.\n\n5. When editing files,
49 | it is easy to accidentally to write code with incorrect indentation or make other
50 | mistakes. Always check the code after you issue an edit to make sure that it reflects
51 | what you wanted to accomplish. If it didn''t, issue another command to fix it.\n\n6.
52 | When editing files, first explain the code you want to edit and why it is causing
53 | the problem. Then explain the edit you want to make and how it fixes the problem.
54 | Explain how the edit does not break existing functionality.\n\n7. Do not try to
55 | install any packages with `pip`, `conda`, or any other way. This will usually not
56 | work. If the environment is not set up correctly, try to fix the issue without executing
57 | python code or running any tests that require the package installed.\n\nSTRATEGY:\n\n1.
58 | Always start by trying to replicate the bug that the issues discusses.\n If the
59 | issue includes code for reproducing the bug, we recommend that you re-implement
60 | that in your environment, and run it to make sure you can reproduce the bug.\n Then
61 | start trying to fix it.\n\n If the bug reproduction script does not print anything
62 | when it successfully runs, we recommend adding a print(\"Script completed successfully,
63 | no errors.\") command at the end of the file,\n so that you can be sure that the
64 | script indeed ran fine all the way through.\n\n2. Locate relevant code using the
65 | find and search commands. `open` the file you want to edit.\n\n3. Use the `edit`
66 | command to perform edits.\n\n4. When you think you''ve fixed the bug, re-run the
67 | bug reproduction script to make sure that the bug has indeed been fixed.\n\n5. Create
68 | additional tests to verify the fix in a style similar to the existing reproduction
69 | script. In particular, make sure to test edge cases.\n If you find any issues,
70 | go back to the file you edited and perform further edits.\n\n(Open file: {{open_file}})\n(Current
71 | directory: {{working_dir}})\nbash-$","next_step_template":"{{observation}}\n(Open
72 | file: {{open_file}})\n(Current directory: {{working_dir}})\nbash-$","next_step_truncated_observation_template":"Observation:
73 | {{observation}}<response clipped><NOTE>Observations should not exceeded {{max_observation_length}}
74 | characters. {{elided_chars}} characters were elided. Please try a different command
75 | that produces less output or use head/tail/grep/redirect the output to a file. Do
76 | not use interactive pagers.</NOTE>","max_observation_length":100000,"next_step_no_output_template":"Your
77 | command ran successfully and did not produce any output.\n(Open file: {{open_file}})\n(Current
78 | directory: {{working_dir}})\nbash-$","strategy_template":null,"demonstration_template":"Here
79 | is a demonstration of how to correctly accomplish this task.\nIt is included to
80 | show you how to correctly use the interface.\nYou do not need to follow exactly
81 | what is done in the demonstration.\n--- DEMONSTRATION ---\n{{demonstration}}\n---
82 | END OF DEMONSTRATION ---\n","demonstrations":["/Users/tosinakinosho/workspaces/mcp-codebase-insight/SWE-agent/trajectories/demonstrations/replay__marshmallow-code__marshmallow-1867__function_calling_replace__install-1/marshmallow-code__marshmallow-1867.traj"],"put_demos_in_history":true,"shell_check_error_template":"Your
83 | bash command contained syntax errors and was NOT executed. Please fix the syntax
84 | errors and try again. This can be the result of not adhering to the syntax for multi-line
85 | commands. Here is the output of `bash -n`:\n{{bash_stdout}}\n{{bash_stderr}}","command_cancelled_timeout_template":"The
86 | command ''{{command}}'' was cancelled because it took more than {{timeout}} seconds.
87 | Please try a different command that completes more quickly."},"tools":{"filter":{"blocklist_error_template":"Operation
88 | ''{{action}}'' is not supported by this environment.","blocklist":["vim","vi","emacs","nano","nohup","gdb","less","tail
89 | -f","python -m venv","make"],"blocklist_standalone":["python","python3","ipython","bash","sh","/bin/bash","/bin/sh","nohup","vi","vim","emacs","nano","su"],"block_unless_regex":{"radare2":"\\b(?:radare2)\\b.*\\s+-c\\s+.*","r2":"\\b(?:radare2)\\b.*\\s+-c\\s+.*"}},"bundles":[{"path":"/Users/tosinakinosho/workspaces/mcp-codebase-insight/SWE-agent/tools/registry","hidden_tools":[]},{"path":"/Users/tosinakinosho/workspaces/mcp-codebase-insight/SWE-agent/tools/defaults","hidden_tools":[]},{"path":"/Users/tosinakinosho/workspaces/mcp-codebase-insight/SWE-agent/tools/search","hidden_tools":[]},{"path":"/Users/tosinakinosho/workspaces/mcp-codebase-insight/SWE-agent/tools/edit_replace","hidden_tools":[]},{"path":"/Users/tosinakinosho/workspaces/mcp-codebase-insight/SWE-agent/tools/submit","hidden_tools":[]}],"env_variables":{"WINDOW":100,"OVERLAP":2},"registry_variables":{},"submit_command":"submit","parse_function":{"error_message":"{%-
90 | if error_code == \"missing\" -%}\nYour last output did not use any tool calls!\nPlease
91 | make sure your output includes exactly _ONE_ function call!\nYou must invoke the
92 | function directly using the function call format.\nYou cannot invoke commands with
93 | ```, you have to use the function call format.\nIf you think you have already resolved
94 | the issue, please submit your changes by running the `submit` command.\nIf you think
95 | you cannot solve the problem, please run `exit_forfeit` (if available) or `submit`.\nElse,
96 | please continue with a new tool call!\n{%- elif error_code == \"multiple\" -%}\nYour
97 | last output included multiple tool calls!\nPlease make sure your output includes
98 | a thought and exactly _ONE_ function call.\n{%- elif error_code == \"unexpected_arg\"
99 | -%}\nYour action could not be parsed properly: {{exception_message}}.\nMake sure
100 | your function call doesn''t include any extra arguments that are not in the allowed
101 | arguments, and only use the allowed commands.\n{%- else -%}\nYour action could not
102 | be parsed properly: {{exception_message}}.\n{% endif %}\n","type":"function_calling"},"enable_bash_tool":true,"format_error_template":"{%-
103 | if error_code == \"missing\" -%}\nYour last output did not use any tool calls!\nPlease
104 | make sure your output includes exactly _ONE_ function call!\nYou must invoke the
105 | function directly using the function call format.\nYou cannot invoke commands with
106 | ```, you have to use the function call format.\nIf you think you have already resolved
107 | the issue, please submit your changes by running the `submit` command.\nIf you think
108 | you cannot solve the problem, please run `exit_forfeit` (if available) or `submit`.\nElse,
109 | please continue with a new tool call!\n{%- elif error_code == \"multiple\" -%}\nYour
110 | last output included multiple tool calls!\nPlease make sure your output includes
111 | a thought and exactly _ONE_ function call.\n{%- elif error_code == \"unexpected_arg\"
112 | -%}\nYour action could not be parsed properly: {{exception_message}}.\nMake sure
113 | your function call doesn''t include any extra arguments that are not in the allowed
114 | arguments, and only use the allowed commands.\n{%- else -%}\nYour action could not
115 | be parsed properly: {{exception_message}}.\n{% endif %}\n","command_docs":"bash:\n docstring:
116 | runs the given command directly in bash\n signature: <command>\n arguments:\n -
117 | command (string) [required]: The bash command to execute.\n\ngoto:\n docstring:
118 | moves the window to show <line_number>\n signature: goto <line_number>\n arguments:\n -
119 | line_number (integer) [required]: the line number to move the window to\n\nopen:\n docstring:
120 | opens the file at the given path in the editor. If line_number is provided, the
121 | window will be move to include that line\n signature: open \"<path>\" [<line_number>]\n arguments:\n -
122 | path (string) [required]: the path to the file to open\n - line_number (integer)
123 | [optional]: the line number to move the window to (if not provided, the window will
124 | start at the top of the file)\n\ncreate:\n docstring: creates and opens a new file
125 | with the given name\n signature: create <filename>\n arguments:\n - filename
126 | (string) [required]: the name of the file to create\n\nscroll_up:\n docstring:
127 | moves the window up 100 lines\n signature: scroll_up\n\nscroll_down:\n docstring:
128 | moves the window down 100 lines\n signature: scroll_down\n\nfind_file:\n docstring:
129 | finds all files with the given name or pattern in dir. If dir is not provided, searches
130 | in the current directory\n signature: find_file <file_name> [<dir>]\n arguments:\n -
131 | file_name (string) [required]: the name of the file or pattern to search for. supports
132 | shell-style wildcards (e.g. *.py)\n - dir (string) [optional]: the directory
133 | to search in (if not provided, searches in the current directory)\n\nsearch_dir:\n docstring:
134 | searches for search_term in all files in dir. If dir is not provided, searches in
135 | the current directory\n signature: search_dir <search_term> [<dir>]\n arguments:\n -
136 | search_term (string) [required]: the term to search for\n - dir (string) [optional]:
137 | the directory to search in (if not provided, searches in the current directory)\n\nsearch_file:\n docstring:
138 | searches for search_term in file. If file is not provided, searches in the current
139 | open file\n signature: search_file <search_term> [<file>]\n arguments:\n -
140 | search_term (string) [required]: the term to search for\n - file (string) [optional]:
141 | the file to search in (if not provided, searches in the current open file)\n\nedit:\n docstring:
142 | Replace first occurrence of <search> with <replace> in the currently displayed lines.
143 | If replace-all is True , replace all occurrences of <search> with <replace>.\nFor
144 | example, if you are looking at this file:\ndef fct():\n print(\"Hello world\")\n\nand
145 | you want to edit the file to read:\ndef fct():\n print(\"Hello\")\n print(\"world\")\n\nyou
146 | can search for `Hello world` and replace with `\"Hello\"\\n print(\"world\")`
147 | (note the extra spaces before the print statement!).\nTips:\n1. Always include proper
148 | whitespace/indentation 2. When you are adding an if/with/try statement, you need
149 | to INDENT the block that follows, so make sure to include it in both your search
150 | and replace strings! 3. If you are wrapping code in a try statement, make sure to
151 | also add an ''except'' or ''finally'' block.\nBefore every edit, please\n1. Explain
152 | the code you want to edit and why it is causing the problem 2. Explain the edit
153 | you want to make and how it fixes the problem 3. Explain how the edit does not break
154 | existing functionality\n\n signature: edit <search> <replace> [<replace-all>]\n\n arguments:\n -
155 | search (string) [required]: the text to search for (make sure to include proper
156 | whitespace if needed)\n - replace (string) [required]: the text to replace the
157 | search with (make sure to include proper whitespace if needed)\n - replace-all
158 | (boolean) [optional]: replace all occurrences rather than the first occurrence within
159 | the displayed lines\n\ninsert:\n docstring: Insert <text> at the end of the currently
160 | opened file or after <line> if specified.\n\n signature: insert <text> [<line>]\n\n arguments:\n -
161 | text (string) [required]: the text to insert\n - line (integer) [optional]: the
162 | line number to insert the text as new lines after\n\nsubmit:\n docstring: submits
163 | the current file\n signature: submit\n\n","multi_line_command_endings":{},"submit_command_end_name":null,"reset_commands":[],"execution_timeout":30,"install_timeout":300,"total_execution_timeout":1800,"max_consecutive_execution_timeouts":3},"history_processors":[{"n":5,"polling":1,"always_remove_output_for_tags":["remove_output"],"always_keep_output_for_tags":["keep_output"],"type":"last_n_observations"}],"model":{"name":"claude-3-5-sonnet-20240620","per_instance_cost_limit":3.0,"total_cost_limit":0.0,"per_instance_call_limit":0,"temperature":0.0,"top_p":1.0,"api_base":null,"api_version":null,"api_key":null,"stop":[],"completion_kwargs":{},"convert_system_to_user":false,"retry":{"retries":20,"min_wait":10.0,"max_wait":120.0},"delay":0.0,"fallbacks":[],"choose_api_key_by_thread":true,"max_input_tokens":null,"max_output_tokens":null},"max_requeries":3,"action_sampler":null,"type":"default"},"problem_statement":{"text":"#
164 | Debug MCP Codebase Insight Tests","extra_fields":{},"type":"text","id":"03565e"},"output_dir":"/Users/tosinakinosho/workspaces/mcp-codebase-insight/trajectories/tosinakinosho/default__claude-3-5-sonnet-20240620__t-0.00__p-1.00__c-3.00___03565e","actions":{"open_pr":false,"pr_config":{"skip_if_commits_reference_issue":true},"apply_patch_locally":false},"env_var_path":null}'
165 |
```
--------------------------------------------------------------------------------
/src/mcp_codebase_insight/core/sse.py:
--------------------------------------------------------------------------------
```python
1 | """Server-Sent Events (SSE) transport implementation for MCP."""
2 |
3 | import asyncio
4 | import logging
5 | import json
6 | from typing import Any, Callable, Dict, List, Optional, Tuple
7 | from datetime import datetime
8 | from starlette.applications import Starlette
9 | from starlette.routing import Mount, Route
10 | from starlette.requests import Request
11 | from starlette.responses import Response, JSONResponse, RedirectResponse, StreamingResponse
12 | import uuid
13 | from anyio.streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream
14 | from starlette.middleware.cors import CORSMiddleware
15 |
16 | from mcp.server.fastmcp import FastMCP
17 | from mcp.server.sse import SseServerTransport
18 | from ..utils.logger import get_logger
19 |
20 | logger = get_logger(__name__)
21 |
22 | async def send_heartbeats(queue: asyncio.Queue, interval: int = 30):
23 | """Send periodic heartbeat messages to keep the connection alive.
24 |
25 | Args:
26 | queue: The queue to send heartbeats to
27 | interval: Time between heartbeats in seconds
28 | """
29 | while True:
30 | try:
31 | await queue.put({"type": "heartbeat", "timestamp": datetime.utcnow().isoformat()})
32 | await asyncio.sleep(interval)
33 | except asyncio.CancelledError:
34 | break
35 | except Exception as e:
36 | logger.error(f"Error sending heartbeat: {e}")
37 | await asyncio.sleep(1) # Brief pause before retrying
38 |
39 | class CodebaseInsightSseTransport(SseServerTransport):
40 | """Custom SSE transport implementation for Codebase Insight."""
41 |
42 | def __init__(self, endpoint: str):
43 | """Initialize the SSE transport.
44 |
45 | Args:
46 | endpoint: The endpoint path for SSE connections
47 | """
48 | super().__init__(endpoint)
49 | self.connections = {}
50 | self.message_queue = asyncio.Queue()
51 | logger.info(f"Initializing SSE transport with endpoint: {endpoint}")
52 |
53 | async def handle_sse(self, request: Request) -> StreamingResponse:
54 | """Handle incoming SSE connection requests.
55 |
56 | Args:
57 | request: The incoming HTTP request
58 |
59 | Returns:
60 | StreamingResponse for the SSE connection
61 | """
62 | connection_id = str(uuid.uuid4())
63 | queue = asyncio.Queue()
64 | self.connections[connection_id] = queue
65 |
66 | logger.info(f"New SSE connection established: {connection_id}")
67 | logger.debug(f"Request headers: {dict(request.headers)}")
68 | logger.debug(f"Active connections: {len(self.connections)}")
69 |
70 | async def event_generator():
71 | try:
72 | logger.debug(f"Starting event generator for connection {connection_id}")
73 | heartbeat_task = asyncio.create_task(send_heartbeats(queue))
74 | logger.debug(f"Heartbeat task started for connection {connection_id}")
75 |
76 | while True:
77 | try:
78 | message = await queue.get()
79 | logger.debug(f"Connection {connection_id} received message: {message}")
80 |
81 | if isinstance(message, dict):
82 | data = json.dumps(message)
83 | else:
84 | data = str(message)
85 |
86 | yield f"data: {data}\n\n"
87 | logger.debug(f"Sent message to connection {connection_id}")
88 |
89 | except asyncio.CancelledError:
90 | logger.info(f"Event generator cancelled for connection {connection_id}")
91 | break
92 | except Exception as e:
93 | logger.error(f"Error in event generator for connection {connection_id}: {e}")
94 | break
95 |
96 | finally:
97 | heartbeat_task.cancel()
98 | try:
99 | await heartbeat_task
100 | except asyncio.CancelledError:
101 | pass
102 |
103 | if connection_id in self.connections:
104 | del self.connections[connection_id]
105 | logger.info(f"Event generator cleaned up for connection {connection_id}")
106 | logger.debug(f"Remaining active connections: {len(self.connections)}")
107 |
108 | return StreamingResponse(
109 | event_generator(),
110 | media_type="text/event-stream",
111 | headers={
112 | "Cache-Control": "no-cache",
113 | "Connection": "keep-alive",
114 | "X-Accel-Buffering": "no",
115 | "Access-Control-Allow-Origin": "*", # Allow CORS
116 | "Access-Control-Allow-Headers": "Content-Type",
117 | "Access-Control-Allow-Methods": "GET, POST"
118 | }
119 | )
120 |
121 | async def handle_message(self, request: Request) -> Response:
122 | """Handle incoming messages to be broadcast over SSE.
123 |
124 | Args:
125 | request: The incoming HTTP request with the message
126 |
127 | Returns:
128 | HTTP response indicating message handling status
129 | """
130 | try:
131 | message = await request.json()
132 |
133 | # Broadcast to all connections
134 | for queue in self.connections.values():
135 | await queue.put(message)
136 |
137 | return JSONResponse({"status": "message sent"})
138 |
139 | except Exception as e:
140 | logger.error(f"Error handling message: {e}")
141 | return JSONResponse(
142 | {"error": str(e)},
143 | status_code=500
144 | )
145 |
146 | async def send(self, message: Any) -> None:
147 | """Send a message to all connected clients.
148 |
149 | Args:
150 | message: The message to send
151 | """
152 | # Put message in queue for all connections
153 | for queue in self.connections.values():
154 | await queue.put(message)
155 |
156 | async def broadcast(self, message: Any) -> None:
157 | """Broadcast a message to all connected clients.
158 |
159 | Args:
160 | message: The message to broadcast
161 | """
162 | await self.send(message)
163 |
164 | async def connect(self) -> Tuple[MemoryObjectReceiveStream, MemoryObjectSendStream]:
165 | """Create a new SSE connection.
166 |
167 | Returns:
168 | Tuple of receive and send streams for the connection
169 | """
170 | # Create memory object streams for this connection
171 | receive_stream = MemoryObjectReceiveStream()
172 | send_stream = MemoryObjectSendStream()
173 |
174 | # Store the connection
175 | connection_id = str(uuid.uuid4())
176 | self.connections[connection_id] = send_stream
177 |
178 | return receive_stream, send_stream
179 |
180 | async def disconnect(self, connection_id: str) -> None:
181 | """Disconnect a client.
182 |
183 | Args:
184 | connection_id: The ID of the connection to disconnect
185 | """
186 | if connection_id in self.connections:
187 | del self.connections[connection_id]
188 | logger.info(f"Disconnected client: {connection_id}")
189 |
190 | async def verify_routes(app: Starlette) -> Dict[str, List[str]]:
191 | """Verify and log all registered routes in the application.
192 |
193 | Args:
194 | app: The Starlette application to verify
195 |
196 | Returns:
197 | Dictionary mapping route paths to their methods
198 | """
199 | routes = {}
200 | for route in app.routes:
201 | if isinstance(route, Mount):
202 | logger.info(f"Mount point: {route.path}")
203 | # Recursively verify mounted routes
204 | mounted_routes = await verify_routes(route.app)
205 | for path, methods in mounted_routes.items():
206 | full_path = f"{route.path}{path}"
207 | routes[full_path] = methods
208 | else:
209 | routes[route.path] = route.methods
210 | logger.info(f"Route: {route.path}, methods: {route.methods}")
211 | return routes
212 |
213 | def create_sse_server(mcp_server: Optional[FastMCP] = None) -> Starlette:
214 | """Create an SSE server instance.
215 |
216 | Args:
217 | mcp_server: Optional FastMCP instance to use. If not provided, a new one will be created.
218 |
219 | Returns:
220 | Starlette application configured for SSE
221 | """
222 | app = Starlette(debug=True) # Enable debug mode for better error reporting
223 |
224 | # Create SSE transport
225 | transport = CodebaseInsightSseTransport("/sse")
226 |
227 | # Add CORS middleware
228 | app.add_middleware(
229 | CORSMiddleware,
230 | allow_origins=["*"], # Allow all origins
231 | allow_credentials=True,
232 | allow_methods=["GET", "POST", "OPTIONS"],
233 | allow_headers=["*"],
234 | expose_headers=["*"]
235 | )
236 |
237 | async def health_check(request: Request) -> JSONResponse:
238 | """Health check endpoint."""
239 | return JSONResponse({
240 | "status": "ok",
241 | "timestamp": datetime.utcnow().isoformat(),
242 | "connections": len(transport.connections)
243 | })
244 |
245 | # Add routes
246 | app.add_route("/health", health_check, methods=["GET"])
247 | app.add_route("/sse", transport.handle_sse, methods=["GET"])
248 | app.add_route("/message", transport.handle_message, methods=["POST"])
249 |
250 | logger.info("Created SSE server with routes:")
251 | asyncio.create_task(verify_routes(app))
252 |
253 | return app
254 |
255 | class MCP_CodebaseInsightServer:
256 | """MCP server implementation for Codebase Insight.
257 |
258 | This class manages the Model Context Protocol server, connecting it to
259 | the Codebase Insight's core components and exposing them as MCP tools.
260 | """
261 |
262 | def __init__(self, server_state):
263 | """Initialize the MCP server with access to the application state.
264 |
265 | Args:
266 | server_state: The global server state providing access to components
267 | """
268 | self.state = server_state
269 | self.mcp_server = FastMCP(name="MCP-Codebase-Insight")
270 | self.tools_registered = False
271 | self._starlette_app = None # Cache the Starlette app
272 | logger.info("MCP Codebase Insight server initialized")
273 |
274 | async def cleanup(self):
275 | """Clean up resources used by the MCP server.
276 |
277 | This method ensures proper shutdown of the MCP server and
278 | releases any resources it might be holding.
279 | """
280 | logger.info("Cleaning up MCP server resources")
281 | # If the MCP server has a shutdown or cleanup method, call it here
282 | # For now, just log the cleanup attempt
283 | self.tools_registered = False
284 | self._starlette_app = None
285 | logger.info("MCP server cleanup completed")
286 |
287 | def is_initialized(self) -> bool:
288 | """Check if the MCP server is properly initialized.
289 |
290 | Returns:
291 | True if the server is initialized and ready to use, False otherwise
292 | """
293 | return self.tools_registered and self._starlette_app is not None
294 |
295 | def register_tools(self):
296 | """Register all available tools with the MCP server.
297 |
298 | This connects the MCP protocol to the Codebase Insight core components,
299 | exposing their functionality through the MCP interface.
300 | """
301 | if self.tools_registered:
302 | logger.debug("Tools already registered, skipping")
303 | return
304 |
305 | logger.info("Registering tools with MCP server")
306 |
307 | # Check if critical dependencies are available
308 | critical_dependencies = ["vector_store", "knowledge_base", "task_manager", "adr_manager"]
309 | missing_dependencies = []
310 |
311 | for dependency in critical_dependencies:
312 | if not self.state.get_component(dependency):
313 | missing_dependencies.append(dependency)
314 |
315 | if missing_dependencies:
316 | logger.warning(f"Some critical dependencies are not available: {', '.join(missing_dependencies)}")
317 | logger.warning("Tools requiring these dependencies will not be registered")
318 | # Don't fail registration completely - continue with available tools
319 |
320 | # Register available tools
321 | try:
322 | self._register_vector_search()
323 | self._register_knowledge()
324 | self._register_adr()
325 | self._register_task()
326 |
327 | # Mark tools as registered even if some failed
328 | self.tools_registered = True
329 | logger.info("MCP tools registration completed")
330 | except Exception as e:
331 | logger.error(f"Error registering MCP tools: {e}", exc_info=True)
332 | # Don't mark as registered if there was an error
333 |
334 | def _register_vector_search(self):
335 | """Register the vector search tool with the MCP server."""
336 | vector_store = self.state.get_component("vector_store")
337 | if not vector_store:
338 | logger.warning("Vector store component not available, skipping tool registration")
339 | return
340 |
341 | # Verify that the vector store is properly initialized
342 | if not hasattr(vector_store, 'search') or not callable(getattr(vector_store, 'search')):
343 | logger.warning("Vector store component does not have a search method, skipping tool registration")
344 | return
345 |
346 | async def vector_search(query: str, limit: int = 5, threshold: float = 0.7,
347 | file_type: Optional[str] = None, path_pattern: Optional[str] = None):
348 | """Search for code snippets semantically similar to the query text."""
349 | logger.debug(f"MCP vector search request: {query=}, {limit=}, {threshold=}")
350 |
351 | # Prepare filters if provided
352 | filter_conditions = {}
353 | if file_type:
354 | filter_conditions["file_type"] = {"$eq": file_type}
355 | if path_pattern:
356 | filter_conditions["path"] = {"$like": path_pattern}
357 |
358 | results = await vector_store.search(
359 | text=query,
360 | filter_conditions=filter_conditions if filter_conditions else None,
361 | limit=limit
362 | )
363 |
364 | # Format results
365 | formatted_results = [
366 | {
367 | "id": result.id,
368 | "score": result.score,
369 | "text": result.metadata.get("text", ""),
370 | "file_path": result.metadata.get("file_path", ""),
371 | "line_range": result.metadata.get("line_range", ""),
372 | "type": result.metadata.get("type", "code"),
373 | "language": result.metadata.get("language", ""),
374 | "timestamp": result.metadata.get("timestamp", "")
375 | }
376 | for result in results
377 | if result.score >= threshold
378 | ]
379 |
380 | return {"results": formatted_results}
381 |
382 | self.mcp_server.add_tool(
383 | name="vector-search",
384 | fn=vector_search,
385 | description="Search for code snippets semantically similar to the query text"
386 | )
387 | logger.debug("Vector search tool registered")
388 |
389 | def _register_knowledge(self):
390 | """Register the knowledge base tool with the MCP server."""
391 | knowledge_base = self.state.get_component("knowledge_base")
392 | if not knowledge_base:
393 | logger.warning("Knowledge base component not available, skipping tool registration")
394 | return
395 |
396 | async def search_knowledge(query: str, pattern_type: str = "code", limit: int = 5):
397 | """Search for patterns in the knowledge base."""
398 | logger.debug(f"MCP knowledge search request: {query=}, {pattern_type=}, {limit=}")
399 |
400 | results = await knowledge_base.search_patterns(
401 | query=query,
402 | pattern_type=pattern_type,
403 | limit=limit
404 | )
405 |
406 | # Format results
407 | formatted_results = [
408 | {
409 | "id": result.id,
410 | "pattern": result.pattern,
411 | "description": result.description,
412 | "type": result.type,
413 | "confidence": result.confidence,
414 | "metadata": result.metadata
415 | }
416 | for result in results
417 | ]
418 |
419 | return {"results": formatted_results}
420 |
421 | self.mcp_server.add_tool(
422 | name="knowledge-search",
423 | fn=search_knowledge,
424 | description="Search for patterns in the knowledge base"
425 | )
426 | logger.debug("Knowledge search tool registered")
427 |
428 | def _register_adr(self):
429 | """Register the ADR management tool with the MCP server."""
430 | adr_manager = self.state.get_component("adr_manager")
431 | if not adr_manager:
432 | logger.warning("ADR manager component not available, skipping tool registration")
433 | return
434 |
435 | async def list_adrs(status: Optional[str] = None, limit: int = 10):
436 | """List architectural decision records."""
437 | logger.debug(f"MCP ADR list request: {status=}, {limit=}")
438 |
439 | try:
440 | adrs = await adr_manager.list_adrs(status=status, limit=limit)
441 |
442 | # Format results
443 | formatted_results = [
444 | {
445 | "id": adr.id,
446 | "title": adr.title,
447 | "status": adr.status,
448 | "date": adr.date.isoformat() if adr.date else None,
449 | "authors": adr.authors,
450 | "summary": adr.summary
451 | }
452 | for adr in adrs
453 | ]
454 |
455 | return {"adrs": formatted_results}
456 | except Exception as e:
457 | logger.error(f"Error listing ADRs: {e}", exc_info=True)
458 | return {"error": str(e), "adrs": []}
459 |
460 | self.mcp_server.add_tool(
461 | name="adr-list",
462 | fn=list_adrs,
463 | description="List architectural decision records"
464 | )
465 | logger.debug("ADR management tool registered")
466 |
467 | def _register_task(self):
468 | """Register the task management tool with the MCP server."""
469 | task_tracker = self.state.get_component("task_tracker")
470 | if not task_tracker:
471 | logger.warning("Task tracker component not available, skipping tool registration")
472 | return
473 |
474 | async def get_task_status(task_id: str):
475 | """Get the status of a specific task."""
476 | logger.debug(f"MCP task status request: {task_id=}")
477 |
478 | try:
479 | status = await task_tracker.get_task_status(task_id)
480 | return status
481 | except Exception as e:
482 | logger.error(f"Error getting task status: {e}", exc_info=True)
483 | return {"error": str(e), "status": "unknown"}
484 |
485 | self.mcp_server.add_tool(
486 | name="task-status",
487 | fn=get_task_status,
488 | description="Get the status of a specific task"
489 | )
490 | logger.debug("Task management tool registered")
491 |
492 | def get_starlette_app(self) -> Starlette:
493 | """Get the Starlette application for the MCP server.
494 |
495 | Returns:
496 | Configured Starlette application
497 | """
498 | # Ensure tools are registered
499 | self.register_tools()
500 |
501 | # Create and return the Starlette app for SSE
502 | if self._starlette_app is None:
503 | self._starlette_app = create_sse_server(self.mcp_server)
504 | return self._starlette_app
505 |
```