This is page 30 of 45. Use http://codebase.md/dicklesworthstone/llm_gateway_mcp_server?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .cursorignore
├── .env.example
├── .envrc
├── .gitignore
├── additional_features.md
├── check_api_keys.py
├── completion_support.py
├── comprehensive_test.py
├── docker-compose.yml
├── Dockerfile
├── empirically_measured_model_speeds.json
├── error_handling.py
├── example_structured_tool.py
├── examples
│ ├── __init__.py
│ ├── advanced_agent_flows_using_unified_memory_system_demo.py
│ ├── advanced_extraction_demo.py
│ ├── advanced_unified_memory_system_demo.py
│ ├── advanced_vector_search_demo.py
│ ├── analytics_reporting_demo.py
│ ├── audio_transcription_demo.py
│ ├── basic_completion_demo.py
│ ├── cache_demo.py
│ ├── claude_integration_demo.py
│ ├── compare_synthesize_demo.py
│ ├── cost_optimization.py
│ ├── data
│ │ ├── sample_event.txt
│ │ ├── Steve_Jobs_Introducing_The_iPhone_compressed.md
│ │ └── Steve_Jobs_Introducing_The_iPhone_compressed.mp3
│ ├── docstring_refiner_demo.py
│ ├── document_conversion_and_processing_demo.py
│ ├── entity_relation_graph_demo.py
│ ├── filesystem_operations_demo.py
│ ├── grok_integration_demo.py
│ ├── local_text_tools_demo.py
│ ├── marqo_fused_search_demo.py
│ ├── measure_model_speeds.py
│ ├── meta_api_demo.py
│ ├── multi_provider_demo.py
│ ├── ollama_integration_demo.py
│ ├── prompt_templates_demo.py
│ ├── python_sandbox_demo.py
│ ├── rag_example.py
│ ├── research_workflow_demo.py
│ ├── sample
│ │ ├── article.txt
│ │ ├── backprop_paper.pdf
│ │ ├── buffett.pdf
│ │ ├── contract_link.txt
│ │ ├── legal_contract.txt
│ │ ├── medical_case.txt
│ │ ├── northwind.db
│ │ ├── research_paper.txt
│ │ ├── sample_data.json
│ │ └── text_classification_samples
│ │ ├── email_classification.txt
│ │ ├── news_samples.txt
│ │ ├── product_reviews.txt
│ │ └── support_tickets.txt
│ ├── sample_docs
│ │ └── downloaded
│ │ └── attention_is_all_you_need.pdf
│ ├── sentiment_analysis_demo.py
│ ├── simple_completion_demo.py
│ ├── single_shot_synthesis_demo.py
│ ├── smart_browser_demo.py
│ ├── sql_database_demo.py
│ ├── sse_client_demo.py
│ ├── test_code_extraction.py
│ ├── test_content_detection.py
│ ├── test_ollama.py
│ ├── text_classification_demo.py
│ ├── text_redline_demo.py
│ ├── tool_composition_examples.py
│ ├── tournament_code_demo.py
│ ├── tournament_text_demo.py
│ ├── unified_memory_system_demo.py
│ ├── vector_search_demo.py
│ ├── web_automation_instruction_packs.py
│ └── workflow_delegation_demo.py
├── LICENSE
├── list_models.py
├── marqo_index_config.json.example
├── mcp_protocol_schema_2025-03-25_version.json
├── mcp_python_lib_docs.md
├── mcp_tool_context_estimator.py
├── model_preferences.py
├── pyproject.toml
├── quick_test.py
├── README.md
├── resource_annotations.py
├── run_all_demo_scripts_and_check_for_errors.py
├── storage
│ └── smart_browser_internal
│ ├── locator_cache.db
│ ├── readability.js
│ └── storage_state.enc
├── test_client.py
├── test_connection.py
├── TEST_README.md
├── test_sse_client.py
├── test_stdio_client.py
├── tests
│ ├── __init__.py
│ ├── conftest.py
│ ├── integration
│ │ ├── __init__.py
│ │ └── test_server.py
│ ├── manual
│ │ ├── test_extraction_advanced.py
│ │ └── test_extraction.py
│ └── unit
│ ├── __init__.py
│ ├── test_cache.py
│ ├── test_providers.py
│ └── test_tools.py
├── TODO.md
├── tool_annotations.py
├── tools_list.json
├── ultimate_mcp_banner.webp
├── ultimate_mcp_logo.webp
├── ultimate_mcp_server
│ ├── __init__.py
│ ├── __main__.py
│ ├── cli
│ │ ├── __init__.py
│ │ ├── __main__.py
│ │ ├── commands.py
│ │ ├── helpers.py
│ │ └── typer_cli.py
│ ├── clients
│ │ ├── __init__.py
│ │ ├── completion_client.py
│ │ └── rag_client.py
│ ├── config
│ │ └── examples
│ │ └── filesystem_config.yaml
│ ├── config.py
│ ├── constants.py
│ ├── core
│ │ ├── __init__.py
│ │ ├── evaluation
│ │ │ ├── base.py
│ │ │ └── evaluators.py
│ │ ├── providers
│ │ │ ├── __init__.py
│ │ │ ├── anthropic.py
│ │ │ ├── base.py
│ │ │ ├── deepseek.py
│ │ │ ├── gemini.py
│ │ │ ├── grok.py
│ │ │ ├── ollama.py
│ │ │ ├── openai.py
│ │ │ └── openrouter.py
│ │ ├── server.py
│ │ ├── state_store.py
│ │ ├── tournaments
│ │ │ ├── manager.py
│ │ │ ├── tasks.py
│ │ │ └── utils.py
│ │ └── ums_api
│ │ ├── __init__.py
│ │ ├── ums_database.py
│ │ ├── ums_endpoints.py
│ │ ├── ums_models.py
│ │ └── ums_services.py
│ ├── exceptions.py
│ ├── graceful_shutdown.py
│ ├── services
│ │ ├── __init__.py
│ │ ├── analytics
│ │ │ ├── __init__.py
│ │ │ ├── metrics.py
│ │ │ └── reporting.py
│ │ ├── cache
│ │ │ ├── __init__.py
│ │ │ ├── cache_service.py
│ │ │ ├── persistence.py
│ │ │ ├── strategies.py
│ │ │ └── utils.py
│ │ ├── cache.py
│ │ ├── document.py
│ │ ├── knowledge_base
│ │ │ ├── __init__.py
│ │ │ ├── feedback.py
│ │ │ ├── manager.py
│ │ │ ├── rag_engine.py
│ │ │ ├── retriever.py
│ │ │ └── utils.py
│ │ ├── prompts
│ │ │ ├── __init__.py
│ │ │ ├── repository.py
│ │ │ └── templates.py
│ │ ├── prompts.py
│ │ └── vector
│ │ ├── __init__.py
│ │ ├── embeddings.py
│ │ └── vector_service.py
│ ├── tool_token_counter.py
│ ├── tools
│ │ ├── __init__.py
│ │ ├── audio_transcription.py
│ │ ├── base.py
│ │ ├── completion.py
│ │ ├── docstring_refiner.py
│ │ ├── document_conversion_and_processing.py
│ │ ├── enhanced-ums-lookbook.html
│ │ ├── entity_relation_graph.py
│ │ ├── excel_spreadsheet_automation.py
│ │ ├── extraction.py
│ │ ├── filesystem.py
│ │ ├── html_to_markdown.py
│ │ ├── local_text_tools.py
│ │ ├── marqo_fused_search.py
│ │ ├── meta_api_tool.py
│ │ ├── ocr_tools.py
│ │ ├── optimization.py
│ │ ├── provider.py
│ │ ├── pyodide_boot_template.html
│ │ ├── python_sandbox.py
│ │ ├── rag.py
│ │ ├── redline-compiled.css
│ │ ├── sentiment_analysis.py
│ │ ├── single_shot_synthesis.py
│ │ ├── smart_browser.py
│ │ ├── sql_databases.py
│ │ ├── text_classification.py
│ │ ├── text_redline_tools.py
│ │ ├── tournament.py
│ │ ├── ums_explorer.html
│ │ └── unified_memory_system.py
│ ├── utils
│ │ ├── __init__.py
│ │ ├── async_utils.py
│ │ ├── display.py
│ │ ├── logging
│ │ │ ├── __init__.py
│ │ │ ├── console.py
│ │ │ ├── emojis.py
│ │ │ ├── formatter.py
│ │ │ ├── logger.py
│ │ │ ├── panels.py
│ │ │ ├── progress.py
│ │ │ └── themes.py
│ │ ├── parse_yaml.py
│ │ ├── parsing.py
│ │ ├── security.py
│ │ └── text.py
│ └── working_memory_api.py
├── unified_memory_system_technical_analysis.md
└── uv.lock
```
# Files
--------------------------------------------------------------------------------
/ultimate_mcp_server/tools/python_sandbox.py:
--------------------------------------------------------------------------------
```python
1 | # ultimate_mcp_server/tools/python_sandbox.py
2 |
3 | """Pyodide-backed sandbox tool for Ultimate MCP Server.
4 |
5 | Provides a secure environment for executing Python code within a headless browser,
6 | with stdout/stderr capture, package management, security controls, and optional REPL functionality.
7 |
8 | Includes integrated offline asset caching for Pyodide.
9 | """
10 |
11 | ###############################################################################
12 | # Standard library & typing
13 | ###############################################################################
14 | import argparse
15 | import asyncio
16 | import atexit
17 | import base64
18 | import collections
19 | import gzip
20 | import hashlib
21 | import json
22 | import logging # Import logging for fallback
23 | import mimetypes
24 | import os
25 | import pathlib
26 | import time
27 | import urllib.error
28 | import urllib.parse
29 | import urllib.request
30 | import uuid
31 | from dataclasses import dataclass, field
32 | from typing import TYPE_CHECKING, Any, Dict, List, Optional, OrderedDict
33 |
34 | # ---------------------------------
35 |
36 | ###############################################################################
37 | # Third‑party – runtime dependency only on Playwright
38 | ###############################################################################
39 | try:
40 | import playwright.async_api as pw
41 |
42 | if TYPE_CHECKING:
43 | # Import SPECIFIC types for type hints inside TYPE_CHECKING
44 | from playwright.async_api import Browser, Page, Request, Route
45 | PLAYWRIGHT_AVAILABLE = True
46 | except ImportError:
47 | pw = None
48 | # Define placeholder types ONLY if playwright is unavailable,
49 | # and inside TYPE_CHECKING if you still want hints to reference *something*
50 | # Although the imports above should handle this for the type checker.
51 | if TYPE_CHECKING:
52 | Browser = Any
53 | Page = Any
54 | Route = Any
55 | Request = Any
56 | PLAYWRIGHT_AVAILABLE = False
57 |
58 | from rich import box
59 | from rich.markup import escape
60 | from rich.panel import Panel
61 | from rich.rule import Rule
62 | from rich.syntax import Syntax
63 | from rich.table import Table
64 |
65 | ###############################################################################
66 | # Project specific imports
67 | ###############################################################################
68 | # Assuming these are correctly located within your project structure
69 | try:
70 | from ultimate_mcp_server.constants import TaskType
71 | from ultimate_mcp_server.exceptions import (
72 | ProviderError,
73 | ToolError,
74 | ToolInputError,
75 | )
76 | from ultimate_mcp_server.tools.base import with_error_handling, with_tool_metrics
77 | from ultimate_mcp_server.utils import get_logger
78 | except ImportError as e:
79 | # Provide a fallback or clearer error if these imports fail
80 | print(f"WARNING: Failed to import Ultimate MCP Server components: {e}")
81 | print("Running in standalone mode or environment misconfiguration.")
82 |
83 | # Define dummy logger/decorators if running standalone for preloading
84 | def get_logger(name):
85 | _logger = logging.getLogger(name)
86 | if not _logger.handlers: # Setup basic config only if no handlers exist
87 | logging.basicConfig(
88 | level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
89 | )
90 | return _logger
91 |
92 | def with_tool_metrics(func):
93 | return func
94 |
95 | def with_error_handling(func):
96 | return func
97 |
98 | # Define dummy exceptions
99 | class ProviderError(Exception):
100 | pass
101 |
102 | class ToolError(Exception):
103 | pass
104 |
105 | class ToolInputError(Exception):
106 | pass
107 |
108 | class TaskType:
109 | CODE_EXECUTION = "code_execution" # Dummy enum value
110 |
111 | from ultimate_mcp_server.utils.logging.console import console
112 |
113 | logger = get_logger("ultimate_mcp_server.tools.python_sandbox")
114 |
115 | # Constant for posting messages back to the sandbox page context
116 | JS_POST_MESSAGE = "(msg) => globalThis.postMessage(msg, '*')"
117 |
118 | ###############################################################################
119 | # Constants & Caching Configuration
120 | ###############################################################################
121 | COMMON_PACKAGES: list[str] = [
122 | "numpy",
123 | "pandas",
124 | "matplotlib",
125 | "scipy",
126 | "networkx",
127 | ]
128 | # Define JSON string *after* COMMON_PACKAGES is defined
129 | COMMON_PACKAGES_JSON = json.dumps(COMMON_PACKAGES)
130 |
131 | MAX_SANDBOXES = 6 # Max number of concurrent browser tabs/sandboxes
132 | GLOBAL_CONCURRENCY = 8 # Max number of simultaneous code executions across all sandboxes
133 | MEM_LIMIT_MB = 512 # Memory limit for the heap watchdog in the browser tab
134 |
135 | # --- Pyodide Version and CDN ---
136 | _PYODIDE_VERSION = "0.27.5" # <<< Ensure this matches the intended version
137 | _CDN_BASE = f"https://cdn.jsdelivr.net/pyodide/v{_PYODIDE_VERSION}/full"
138 | # Note: PYODIDE_CDN variable might not be strictly necessary if importing .mjs directly
139 | PYODIDE_CDN = f"{_CDN_BASE}/pyodide.js"
140 |
141 | # --- Define the packages to be loaded AT STARTUP ---
142 | # These will be baked into the loadPyodide call via the template
143 | CORE_PACKAGES_TO_LOAD_AT_STARTUP: list[str] = [
144 | "numpy",
145 | "pandas",
146 | "matplotlib",
147 | "scipy",
148 | "networkx",
149 | "micropip", # Good to include if you often load wheels later
150 | ]
151 | # Generate the JSON string to be injected into the HTML template
152 | CORE_PACKAGES_JSON_FOR_TEMPLATE = json.dumps(CORE_PACKAGES_TO_LOAD_AT_STARTUP)
153 |
154 | # --- Asset Caching Configuration ---
155 | _CACHE_DIR = (
156 | pathlib.Path(os.getenv("XDG_CACHE_HOME", "~/.cache")).expanduser()
157 | / "ultimate_mcp_server"
158 | / "pyodide"
159 | / _PYODIDE_VERSION # Versioned cache directory
160 | )
161 | try:
162 | _CACHE_DIR.mkdir(parents=True, exist_ok=True)
163 | logger.info(f"Using Pyodide asset cache directory: {_CACHE_DIR}")
164 | except OSError as e:
165 | logger.error(
166 | f"Failed to create Pyodide asset cache directory {_CACHE_DIR}: {e}. Caching might fail."
167 | )
168 |
169 | ################################################################################
170 | # Diagnostic logging helpers
171 | ################################################################################
172 | # level 0 = quiet, 1 = basic req/resp, 2 = full body/hex dumps
173 | _VERBOSE_SANDBOX_LOGGING = int(os.getenv("PYODIDE_SANDBOX_DEBUG", "0") or 0)
174 |
175 | def _wire_page_logging(page: "Page", session_id: str) -> None: # type: ignore
176 | """
177 | Mirrors everything interesting coming out of the browser tab back into our
178 | Python logger. When PYODIDE_SANDBOX_DEBUG=2 we also dump request/response
179 | headers and first 64 bytes of every body.
180 | """
181 |
182 | # ───────── console / JS errors ───────────────────────────────────────────
183 | def _log_console(msg):
184 | try:
185 | # Safely access properties, defaulting if necessary
186 | lvl = msg.type if not callable(getattr(msg, "type", None)) else msg.type()
187 | txt = msg.text if not callable(getattr(msg, "text", None)) else msg.text()
188 | loc = msg.location if not callable(getattr(msg, "location", None)) else msg.location()
189 |
190 | src = ""
191 | if isinstance(loc, dict):
192 | src = f"{loc.get('url', '')}:{loc.get('lineNumber', '?')}:{loc.get('columnNumber', '?')}"
193 | elif loc:
194 | src = str(loc)
195 |
196 | line = f"SB[{session_id}] {src} ▶ {txt}" if src else f"SB[{session_id}] ▶ {txt}"
197 |
198 | log_func = {
199 | "error": logger.error,
200 | "warning": logger.warning,
201 | "warn": logger.warning,
202 | "info": logger.info,
203 | "log": logger.info,
204 | "debug": logger.debug,
205 | "trace": logger.debug,
206 | }.get(str(lvl).lower(), logger.debug)
207 |
208 | log_func(line)
209 | except Exception as e:
210 | logger.error(f"SB[{session_id}] Error in console message processing: {e}")
211 |
212 | try:
213 | page.on("console", _log_console)
214 | page.on(
215 | "pageerror",
216 | lambda e: logger.error(f"SB[{session_id}] PageError ▶ {e.message}\n{e.stack}"),
217 | )
218 | page.on("crash", lambda: logger.critical(f"SB[{session_id}] **PAGE CRASHED**"))
219 | except Exception as e:
220 | logger.error(f"SB[{session_id}] Failed to attach basic page log listeners: {e}")
221 |
222 | # ───────── high-level net trace ─────────────────────────────────────────
223 | if _VERBOSE_SANDBOX_LOGGING > 0:
224 | try:
225 | page.on("request", lambda r: logger.debug(f"SB[{session_id}] → {r.method} {r.url}"))
226 | page.on(
227 | "requestfailed",
228 | lambda r: logger.warning(f"SB[{session_id}] ✗ {r.method} {r.url} ▶ {r.failure}"),
229 | )
230 |
231 | async def _resp_logger(resp: "pw.Response"): # type: ignore # Use string literal hint
232 | try:
233 | status = resp.status
234 | url = resp.url
235 | if status == 200 and url.startswith("data:") and _VERBOSE_SANDBOX_LOGGING < 2:
236 | return
237 |
238 | # Use resp.all_headers() which returns a dict directly
239 | hdrs = await resp.all_headers()
240 | ce = hdrs.get("content-encoding", "")
241 | ct = hdrs.get("content-type", "")
242 | log_line = (
243 | f"SB[{session_id}] ← {status} {url} (type='{ct}', enc='{ce or 'none'}')"
244 | )
245 |
246 | if _VERBOSE_SANDBOX_LOGGING > 1 or status >= 400: # Log body for errors too
247 | try:
248 | body = await resp.body()
249 | sig = body[:64]
250 | hexs = " ".join(f"{b:02x}" for b in sig)
251 | log_line += f" (len={len(body)}, first-64: {hexs})"
252 | except Exception as body_err:
253 | # Handle cases where body might not be available (e.g., redirects)
254 | log_line += f" (body unavailable: {body_err})"
255 | logger.debug(log_line)
256 | except Exception as e:
257 | logger.warning(f"SB[{session_id}] Error in response logger: {e}")
258 |
259 | page.on("response", lambda r: asyncio.create_task(_resp_logger(r)))
260 | except Exception as e:
261 | logger.error(f"SB[{session_id}] Failed to attach network trace log listeners: {e}")
262 |
263 |
264 | ###############################################################################
265 | # Asset Caching Helper Functions (Integrated)
266 | ###############################################################################
267 |
268 |
269 | def _local_path(remote_url: str) -> pathlib.Path:
270 | """Generates the local cache path for a given remote URL."""
271 | try:
272 | parsed_url = urllib.parse.urlparse(remote_url)
273 | path_part = parsed_url.path if parsed_url.path else "/"
274 | fname = pathlib.Path(path_part).name
275 | if not fname or fname == "/":
276 | fname = hashlib.md5(remote_url.encode()).hexdigest() + ".cache"
277 | logger.debug(
278 | f"No filename in path '{path_part}', using hash '{fname}' for {remote_url}"
279 | )
280 | except Exception as e:
281 | logger.warning(f"Error parsing URL '{remote_url}' for filename: {e}. Falling back to hash.")
282 | fname = hashlib.md5(remote_url.encode()).hexdigest() + ".cache"
283 |
284 | return _CACHE_DIR / fname
285 |
286 |
287 | def _fetch_asset_sync(remote_url: str, max_age_s: int = 7 * 24 * 3600) -> bytes:
288 | """
289 | Synchronous version: Return requested asset from cache or download.
290 | Used by Playwright interceptor and preloader.
291 | """
292 | p = _local_path(remote_url)
293 | use_cache = False
294 | if p.exists():
295 | try:
296 | file_stat = p.stat()
297 | file_age = time.time() - file_stat.st_mtime
298 | if file_age < max_age_s:
299 | if file_stat.st_size > 0:
300 | logger.debug(
301 | f"[Cache] HIT for {remote_url} (age: {file_age:.0f}s < {max_age_s}s)"
302 | )
303 | use_cache = True
304 | else:
305 | logger.warning(
306 | f"[Cache] Hit for {remote_url}, but file is empty. Re-downloading."
307 | )
308 | else:
309 | logger.info(
310 | f"[Cache] STALE for {remote_url} (age: {file_age:.0f}s >= {max_age_s}s)"
311 | )
312 | except OSError as e:
313 | logger.warning(f"[Cache] Error accessing cache file {p}: {e}. Will attempt download.")
314 |
315 | if use_cache:
316 | try:
317 | return p.read_bytes()
318 | except OSError as e:
319 | logger.warning(f"[Cache] Error reading cached file {p}: {e}. Will attempt download.")
320 |
321 | logger.info(f"[Cache] MISS or STALE/Error for {remote_url}. Downloading...")
322 | downloaded_data = None
323 | try:
324 | req = urllib.request.Request(
325 | remote_url,
326 | headers={"User-Agent": "UltimateMCPServer-AssetCache/1.0", "Accept-Encoding": "gzip"},
327 | )
328 | with urllib.request.urlopen(req, timeout=30) as resp:
329 | if resp.status != 200:
330 | raise urllib.error.HTTPError(
331 | remote_url, resp.status, resp.reason, resp.headers, None
332 | )
333 | downloaded_data = resp.read()
334 | # Handle potential gzip encoding from server
335 | if resp.headers.get("Content-Encoding") == "gzip":
336 | try:
337 | downloaded_data = gzip.decompress(downloaded_data)
338 | logger.debug(f"[Cache] Decompressed gzip response for {remote_url}")
339 | except gzip.BadGzipFile:
340 | logger.warning(
341 | f"[Cache] Received gzip header but invalid gzip data for {remote_url}. Using raw."
342 | )
343 | except Exception as gz_err:
344 | logger.warning(
345 | f"[Cache] Error decompressing gzip for {remote_url}: {gz_err}. Using raw."
346 | )
347 |
348 | logger.info(
349 | f"[Cache] Downloaded {len(downloaded_data)} bytes from {remote_url} (status: {resp.status})"
350 | )
351 |
352 | except urllib.error.HTTPError as e:
353 | logger.warning(f"[Cache] HTTP error downloading {remote_url}: {e.code} {e.reason}")
354 | if p.exists():
355 | try:
356 | stale_stat = p.stat()
357 | if stale_stat.st_size > 0:
358 | logger.warning(
359 | f"[Cache] Using STALE cache file {p} as fallback due to HTTP {e.code}."
360 | )
361 | return p.read_bytes()
362 | except OSError as read_err:
363 | logger.error(
364 | f"[Cache] Failed reading fallback cache {p} after download error: {read_err}"
365 | )
366 | raise RuntimeError(
367 | f"Cannot download {remote_url} (HTTP {e.code}) and no usable cache available"
368 | ) from e
369 | except urllib.error.URLError as e:
370 | logger.warning(f"[Cache] Network error downloading {remote_url}: {e.reason}")
371 | if p.exists():
372 | try:
373 | stale_stat = p.stat()
374 | if stale_stat.st_size > 0:
375 | logger.warning(
376 | f"[Cache] Using STALE cache file {p} as fallback due to network error."
377 | )
378 | return p.read_bytes()
379 | except OSError as read_err:
380 | logger.error(
381 | f"[Cache] Failed reading fallback cache {p} after network error: {read_err}"
382 | )
383 | raise RuntimeError(
384 | f"Cannot download {remote_url} ({e.reason}) and no usable cache available"
385 | ) from e
386 | except Exception as e:
387 | logger.error(f"[Cache] Unexpected error downloading {remote_url}: {e}", exc_info=True)
388 | if p.exists():
389 | try:
390 | stale_stat = p.stat()
391 | if stale_stat.st_size > 0:
392 | logger.warning(
393 | f"[Cache] Using STALE cache file {p} as fallback due to unexpected error."
394 | )
395 | return p.read_bytes()
396 | except OSError as read_err:
397 | logger.error(
398 | f"[Cache] Failed reading fallback cache {p} after unexpected error: {read_err}"
399 | )
400 | raise RuntimeError(
401 | f"Cannot download {remote_url} (unexpected error: {e}) and no usable cache available"
402 | ) from e
403 |
404 | if downloaded_data is not None:
405 | try:
406 | tmp_suffix = f".tmp_{os.getpid()}_{uuid.uuid4().hex[:6]}"
407 | tmp_path = p.with_suffix(p.suffix + tmp_suffix)
408 | tmp_path.write_bytes(downloaded_data)
409 | tmp_path.replace(p)
410 | logger.info(f"[Cache] Saved {len(downloaded_data)} bytes for {remote_url} to {p}")
411 | except OSError as e:
412 | logger.error(f"[Cache] Failed write cache file {p}: {e}")
413 | return downloaded_data
414 | else:
415 | raise RuntimeError(f"Download completed for {remote_url} but data is None (internal error)")
416 |
417 |
418 | ###############################################################################
419 | # Browser / bookkeeping singletons
420 | ###############################################################################
421 | _BROWSER: Optional["Browser"] = None # type: ignore # Use string literal hint
422 | _PAGES: OrderedDict[str, "PyodideSandbox"] = collections.OrderedDict()
423 | _GLOBAL_SEM: Optional[asyncio.Semaphore] = None
424 |
425 |
426 | ###############################################################################
427 | # PyodideSandbox Class Definition
428 | ###############################################################################
429 | @dataclass(slots=True)
430 | class PyodideSandbox:
431 | """One Chromium tab with Pyodide runtime (optionally persistent)."""
432 |
433 | page: "Page" # type: ignore # Use string literal hint
434 | allow_network: bool = False
435 | allow_fs: bool = False
436 | ready_evt: asyncio.Event = field(default_factory=asyncio.Event)
437 | created_at: float = field(default_factory=time.time)
438 | last_used: float = field(default_factory=time.time)
439 | _init_timeout: int = 90
440 | _message_handlers: Dict[str, asyncio.Queue] = field(default_factory=dict)
441 | _init_queue: asyncio.Queue = field(default_factory=asyncio.Queue)
442 |
443 | async def init(self):
444 | """Load boot HTML, set up messaging (direct callback), and wait for ready signal."""
445 | if not PLAYWRIGHT_AVAILABLE:
446 | raise RuntimeError("Playwright is not installed. Cannot initialize sandbox.")
447 |
448 | logger.info(f"Initializing PyodideSandbox instance (Page: {self.page.url})...")
449 | init_start_time = time.monotonic()
450 |
451 | # === 1. Network Interception Setup ===
452 | logger.debug("Setting up network interception...")
453 | try:
454 | # Define the interception logic inline or call an external helper
455 | cdn_base_lower = _CDN_BASE.lower()
456 |
457 | async def _block(route: "Route", request: "Request"): # type: ignore
458 | url = request.url
459 | low = url.lower()
460 | is_cdn = low.startswith(cdn_base_lower)
461 | is_pypi = "pypi.org/simple" in low or "files.pythonhosted.org" in low
462 |
463 | if is_cdn:
464 | try:
465 | # Assuming _fetch_asset_sync is correctly defined elsewhere
466 | body = _fetch_asset_sync(url)
467 | ctype = mimetypes.guess_type(url)[0] or "application/octet-stream"
468 | headers = {
469 | "Content-Type": ctype,
470 | "Access-Control-Allow-Origin": "*",
471 | "Cache-Control": "public, max-age=31536000",
472 | }
473 | # Simple check for gzip magic bytes; don't decompress here, let browser handle it
474 | if body.startswith(b"\x1f\x8b"):
475 | headers["Content-Encoding"] = "gzip"
476 |
477 | if _VERBOSE_SANDBOX_LOGGING > 1:
478 | logger.debug(
479 | f"[Intercept] FULFILL CDN {url} (type={ctype}, enc={headers.get('Content-Encoding', 'none')}, len={len(body)})"
480 | )
481 | await route.fulfill(status=200, body=body, headers=headers)
482 | return
483 | except Exception as exc:
484 | logger.error(
485 | f"[Intercept] FAILED serving CDN {url} from cache/download: {exc}",
486 | exc_info=_VERBOSE_SANDBOX_LOGGING > 1,
487 | )
488 | await route.abort(error_code="failed")
489 | return
490 |
491 | # Allow PyPI only if explicitly enabled
492 | if self.allow_network and is_pypi:
493 | if _VERBOSE_SANDBOX_LOGGING > 0:
494 | logger.debug(f"[Intercept] ALLOW PyPI {url}")
495 | try:
496 | await route.continue_()
497 | except Exception as cont_err:
498 | logger.warning(
499 | f"[Intercept] Error continuing PyPI request {url}: {cont_err}"
500 | )
501 | try:
502 | await route.abort(error_code="failed")
503 | except Exception:
504 | pass
505 | return
506 |
507 | # Block other network requests by default
508 | # Log less aggressively for common browser noise
509 | if not any(low.endswith(ext) for ext in [".ico", ".png", ".woff", ".woff2"]):
510 | if _VERBOSE_SANDBOX_LOGGING > 0:
511 | logger.debug(f"[Intercept] BLOCK {url}")
512 | try:
513 | await route.abort(error_code="blockedbyclient")
514 | except Exception:
515 | pass # Ignore errors aborting (e.g., already handled)
516 |
517 | await self.page.route("**/*", _block)
518 | logger.info("Network interception active.")
519 | except Exception as e:
520 | logger.error(f"Failed to set up network interception: {e}", exc_info=True)
521 | await self._try_close_page("Network Intercept Setup Error")
522 | raise ToolError(f"Failed to configure sandbox network rules: {e}") from e
523 |
524 | # === 2. Load Boot HTML ===
525 | logger.debug("Loading boot HTML template...")
526 | try:
527 | template_path = pathlib.Path(__file__).parent / "pyodide_boot_template.html"
528 | if not template_path.is_file():
529 | raise FileNotFoundError(f"Boot template not found at {template_path}")
530 | boot_html_template = template_path.read_text(encoding="utf-8")
531 |
532 | # Replace placeholders, including the CORE packages JSON
533 | processed_boot_html = (
534 | boot_html_template.replace("__CDN_BASE__", _CDN_BASE)
535 | .replace("__PYODIDE_VERSION__", _PYODIDE_VERSION)
536 | # *** Use the new constant for core packages ***
537 | .replace("__CORE_PACKAGES_JSON__", CORE_PACKAGES_JSON_FOR_TEMPLATE)
538 | .replace("__MEM_LIMIT_MB__", str(MEM_LIMIT_MB)) # Keep MEM_LIMIT if using watchdog
539 | )
540 |
541 | # Check essential placeholders
542 | essential_placeholders = ["__CDN_BASE__", "__PYODIDE_VERSION__"]
543 | # Check optional placeholders based on template features
544 | optional_placeholders = ["__CORE_PACKAGES_JSON__", "__MEM_LIMIT_MB__"]
545 | missing_essential = [p for p in essential_placeholders if p in processed_boot_html]
546 | missing_optional = [p for p in optional_placeholders if p in processed_boot_html]
547 |
548 | if missing_essential:
549 | logger.critical(
550 | f"CRITICAL: Essential placeholders missing in boot HTML: {missing_essential}. Aborting."
551 | )
552 | raise ToolError(
553 | f"Essential placeholders missing in boot template: {missing_essential}"
554 | )
555 | if missing_optional:
556 | logger.warning(
557 | f"Optional placeholders missing in boot HTML: {missing_optional}. Check template if features are expected."
558 | )
559 |
560 | await self.page.set_content(
561 | processed_boot_html,
562 | wait_until="domcontentloaded",
563 | timeout=60000, # Slightly longer timeout for package loading
564 | )
565 | logger.info("Boot HTML loaded into page.")
566 | except FileNotFoundError as e:
567 | logger.error(f"Failed to load boot HTML template: {e}", exc_info=True)
568 | await self._try_close_page("Boot HTML Template Not Found")
569 | raise ToolError(f"Could not find sandbox boot HTML template: {e}") from e
570 | except Exception as e:
571 | logger.error(f"Failed loading boot HTML content: {e}", exc_info=True)
572 | await self._try_close_page("Boot HTML Load Error")
573 | raise ToolError(f"Could not load sandbox boot HTML content: {e}") from e
574 |
575 | # === 3. Setup Communication Channels ===
576 | # This involves two parts:
577 | # a) Exposing a Python function for JS to send *replies* directly.
578 | # b) Exposing a Python function for JS to send the initial *ready/error* signal.
579 |
580 | # --- 3a. Setup for Execution Replies ---
581 | logger.debug("Setting up direct reply mechanism (JS->Python)...")
582 | try:
583 | # This Python function will be called by JavaScript's `window._deliverReplyToHost(reply)`
584 | async def _deliver_reply_to_host(payload: Any):
585 | msg_id = None # Define outside try block
586 | try:
587 | if not isinstance(payload, dict):
588 | if _VERBOSE_SANDBOX_LOGGING > 1:
589 | logger.debug(f"Host received non-dict reply payload: {type(payload)}")
590 | return
591 | data = payload
592 | msg_id = data.get("id")
593 | if not msg_id:
594 | logger.warning(f"Host received reply payload without an ID: {data}")
595 | return
596 |
597 | # Log received reply
598 | if _VERBOSE_SANDBOX_LOGGING > 0:
599 | log_detail = (
600 | f"ok={data.get('ok')}"
601 | if _VERBOSE_SANDBOX_LOGGING == 1
602 | else json.dumps(data, default=str)
603 | )
604 | logger.debug(
605 | f"Host received reply via exposed function (id: {msg_id}): {log_detail}"
606 | )
607 |
608 | # Route reply to the waiting asyncio Queue in _message_handlers
609 | if msg_id in self._message_handlers:
610 | await self._message_handlers[msg_id].put(data)
611 | if _VERBOSE_SANDBOX_LOGGING > 0:
612 | logger.debug(f"Reply payload for ID {msg_id} routed.")
613 | elif _VERBOSE_SANDBOX_LOGGING > 0:
614 | logger.debug(
615 | f"Host received reply for unknown/stale execution ID: {msg_id}"
616 | )
617 |
618 | except Exception as e:
619 | logger.error(
620 | f"Error processing execution reply payload (id: {msg_id or 'unknown'}) from sandbox: {e}",
621 | exc_info=True,
622 | )
623 |
624 | reply_handler_name = "_deliverReplyToHost" # Must match the name called in JS template
625 | await self.page.expose_function(reply_handler_name, _deliver_reply_to_host)
626 | logger.info(f"Python function '{reply_handler_name}' exposed for JS execution replies.")
627 |
628 | except Exception as e:
629 | logger.error(f"Failed to expose reply handler function: {e}", exc_info=True)
630 | await self._try_close_page("Reply Handler Setup Error")
631 | raise ToolError(f"Could not expose reply handler to sandbox: {e}") from e
632 |
633 | # --- 3b. Setup for Initial Ready/Error Signal ---
634 | # The JS template sends the initial 'pyodide_ready' or 'pyodide_init_error' via postMessage.
635 | # We need a way to capture *only* that specific message and put it on _init_queue.
636 | logger.debug("Setting up listener for initial ready/error signal (JS->Python)...")
637 | try:
638 | # This Python function will be called by the JS listener below
639 | async def _handle_initial_message(payload: Any):
640 | try:
641 | if not isinstance(payload, dict):
642 | return # Ignore non-dicts
643 | msg_id = payload.get("id")
644 | if msg_id == "pyodide_ready" or msg_id == "pyodide_init_error":
645 | log_level = logger.info if payload.get("ready") else logger.error
646 | log_level(
647 | f"Received initial status message from sandbox via exposed function: {payload}"
648 | )
649 | await self._init_queue.put(payload) # Put it on the init queue
650 | # Optionally remove the listener after receiving the first signal? Might be risky.
651 | # Ignore other messages potentially caught by this listener
652 | except Exception as e:
653 | logger.error(
654 | f"Error processing initial message from sandbox: {e}", exc_info=True
655 | )
656 | # Put an error onto the queue to unblock init waiter
657 | await self._init_queue.put(
658 | {
659 | "id": "pyodide_init_error",
660 | "ok": False,
661 | "error": {
662 | "type": "HostProcessingError",
663 | "message": f"Error handling init message: {e}",
664 | },
665 | }
666 | )
667 |
668 | init_handler_name = "_handleInitialMessage"
669 | await self.page.expose_function(init_handler_name, _handle_initial_message)
670 |
671 | # Evaluate JavaScript to add a *specific* listener that calls the exposed init handler
672 | await self.page.evaluate(f"""
673 | console.log('[PyodideBoot] Adding specific listener for initial ready/error messages...');
674 | // Ensure we don't add multiple listeners if init is somehow re-run
675 | if (!window._initialMessageListenerAdded) {{
676 | window.addEventListener('message', (event) => {{
677 | const data = event.data;
678 | // Check if the exposed function exists and if it's the specific message we want
679 | if (typeof window.{init_handler_name} === 'function' &&
680 | typeof data === 'object' && data !== null &&
681 | (data.id === 'pyodide_ready' || data.id === 'pyodide_init_error'))
682 | {{
683 | // Forward only specific initial messages to the exposed Python function
684 | console.log('[PyodideBoot] Forwarding initial message to host:', data.id);
685 | window.{init_handler_name}(data);
686 | }}
687 | }});
688 | window._initialMessageListenerAdded = true; // Flag to prevent multiple adds
689 | console.log('[PyodideBoot] Initial message listener added.');
690 | }} else {{
691 | console.log('[PyodideBoot] Initial message listener already added.');
692 | }}
693 | """)
694 | logger.info(
695 | f"Python function '{init_handler_name}' exposed and JS listener added for initial signal."
696 | )
697 |
698 | except Exception as e:
699 | logger.error(f"Failed to set up initial signal listener: {e}", exc_info=True)
700 | await self._try_close_page("Initial Signal Listener Setup Error")
701 | raise ToolError(f"Could not set up initial signal listener: {e}") from e
702 |
703 | # === 4. Wait for Ready Signal ===
704 | logger.info(f"Waiting for sandbox ready signal (timeout: {self._init_timeout}s)...")
705 | try:
706 | # Wait for a message to appear on the _init_queue
707 | init_data = await asyncio.wait_for(self._init_queue.get(), timeout=self._init_timeout)
708 |
709 | # Check the content of the message
710 | if init_data.get("id") == "pyodide_init_error" or init_data.get("ok") is False:
711 | error_details = init_data.get(
712 | "error", {"message": "Unknown initialization error reported by sandbox."}
713 | )
714 | error_msg = error_details.get("message", "Unknown Error")
715 | logger.error(f"Pyodide sandbox initialization failed inside browser: {error_msg}")
716 | await self._try_close_page("Initialization Error Reported by JS")
717 | raise ToolError(f"Pyodide sandbox initialization failed: {error_msg}")
718 |
719 | if not init_data.get("ready"):
720 | logger.error(f"Received unexpected init message without 'ready' flag: {init_data}")
721 | await self._try_close_page("Unexpected Init Message from JS")
722 | raise ToolError("Received unexpected initialization message from sandbox.")
723 |
724 | # If we received the correct ready message
725 | self.ready_evt.set() # Set the event flag
726 | boot_ms_reported = init_data.get("boot_ms", "N/A")
727 | init_duration = time.monotonic() - init_start_time
728 | logger.info(
729 | f"Pyodide sandbox ready signal received (reported boot: {boot_ms_reported}ms, total init wait: {init_duration:.2f}s)"
730 | )
731 |
732 | except asyncio.TimeoutError as e:
733 | logger.error(f"Timeout ({self._init_timeout}s) waiting for Pyodide ready signal.")
734 | await self._check_page_responsiveness(
735 | "Timeout Waiting for Ready"
736 | ) # Check if page is stuck
737 | await self._try_close_page("Timeout Waiting for Ready")
738 | raise ToolError(
739 | f"Sandbox failed to initialize within timeout ({self._init_timeout}s)."
740 | ) from e
741 | except Exception as e:
742 | # Catch other errors during the wait/processing phase
743 | logger.error(f"Error during sandbox initialization wait: {e}", exc_info=True)
744 | await self._try_close_page("Initialization Wait Error")
745 | if isinstance(e, ToolError): # Don't wrap existing ToolErrors
746 | raise e
747 | raise ToolError(f"Unexpected error during sandbox initialization wait: {e}") from e
748 |
749 | async def _check_page_responsiveness(self, context: str) -> bool: # Return boolean
750 | """Tries to evaluate a simple JS command to check if the page is alive."""
751 | if self.page and not self.page.is_closed():
752 | try:
753 | await asyncio.wait_for(self.page.evaluate("1+1"), timeout=5.0)
754 | logger.debug(f"Page responded after {context}.")
755 | return True
756 | except Exception as page_err:
757 | logger.error(f"Page seems unresponsive after {context}: {page_err}")
758 | return False
759 | else:
760 | logger.debug(
761 | f"Page already closed or non-existent during responsiveness check ({context})."
762 | )
763 | return False # Not responsive if closed
764 |
765 | async def _try_close_page(self, reason: str):
766 | """Attempts to close the sandbox page, logging errors."""
767 | if self.page and not self.page.is_closed():
768 | logger.info(f"Attempting to close sandbox page due to: {reason}")
769 | try:
770 | await self.page.close()
771 | logger.info(f"Sandbox page closed successfully after {reason}.")
772 | except Exception as close_err:
773 | logger.warning(f"Error closing page after {reason}: {close_err}")
774 | else:
775 | logger.debug(
776 | f"Page already closed or non-existent when trying to close due to: {reason}"
777 | )
778 |
779 | async def execute(
780 | self,
781 | code: str,
782 | packages: list[str] | None,
783 | wheels: list[str] | None,
784 | timeout_ms: int,
785 | repl_mode: bool = False,
786 | ) -> Dict[str, Any]:
787 | """Sends code to the sandbox for execution and returns the result."""
788 | if not PLAYWRIGHT_AVAILABLE:
789 | # This condition should ideally be checked before creating/getting the sandbox
790 | # but is included here for robustness.
791 | raise ToolError("Playwright is not installed.")
792 | if not self.page or self.page.is_closed():
793 | raise ToolError("Cannot execute code: Sandbox page is closed.")
794 | if not self.ready_evt.is_set():
795 | # Wait briefly for the ready event if it's not set yet, in case of race conditions
796 | try:
797 | await asyncio.wait_for(self.ready_evt.wait(), timeout=1.0)
798 | except asyncio.TimeoutError as e:
799 | raise ToolError(
800 | "Cannot execute code: Sandbox is not ready (or timed out becoming ready)."
801 | ) from e
802 |
803 | self.last_used = time.time()
804 | global _GLOBAL_SEM
805 | if _GLOBAL_SEM is None:
806 | # Initialize if it hasn't been already (should be done in _get_sandbox, but safety check)
807 | logger.warning("Global execution semaphore not initialized, initializing now.")
808 | _GLOBAL_SEM = asyncio.Semaphore(GLOBAL_CONCURRENCY)
809 |
810 | # Acquire the semaphore to limit concurrency across all sandboxes
811 | async with _GLOBAL_SEM:
812 | exec_id = f"exec-{uuid.uuid4().hex[:8]}"
813 | response_queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue()
814 | self._message_handlers[exec_id] = response_queue
815 |
816 | try:
817 | # Encode the user's Python code to Base64
818 | code_b64 = base64.b64encode(code.encode("utf-8")).decode("ascii")
819 | except Exception as enc_err:
820 | # If encoding fails, it's an input error, no need to involve the sandbox
821 | self._message_handlers.pop(exec_id, None) # Clean up handler
822 | raise ToolInputError(f"Failed to encode code to base64: {enc_err}") from enc_err
823 |
824 | # Prepare the message payload for the JavaScript side
825 | payload = {
826 | "type": "exec",
827 | "id": exec_id,
828 | "code_b64": code_b64,
829 | "packages": packages or [],
830 | "wheels": wheels or [],
831 | "repl_mode": repl_mode,
832 | }
833 | data: dict[str, Any] = {} # Initialize response data dictionary
834 |
835 | try:
836 | logger.debug(
837 | f"Sending execution request to sandbox (id: {exec_id}, repl={repl_mode})"
838 | )
839 | # Send the message to the sandbox page's window context
840 | await self.page.evaluate("window.postMessage", payload)
841 |
842 | logger.debug(
843 | f"Waiting for execution result (id: {exec_id}, timeout: {timeout_ms}ms)..."
844 | )
845 | # Wait for the response message from the sandbox via the queue
846 | data = await asyncio.wait_for(response_queue.get(), timeout=timeout_ms / 1000.0)
847 | logger.debug(f"Received execution result (id: {exec_id}): ok={data.get('ok')}")
848 |
849 | except asyncio.TimeoutError:
850 | logger.warning(
851 | f"Execution timed out waiting for response (id: {exec_id}, timeout: {timeout_ms}ms)"
852 | )
853 | # Check if the page is still responsive after the timeout
854 | await self._check_page_responsiveness(f"Timeout id={exec_id}")
855 | # Return a structured timeout error
856 | return {
857 | "ok": False,
858 | "error": {
859 | "type": "TimeoutError",
860 | "message": f"Execution timed out after {timeout_ms}ms waiting for sandbox response.",
861 | "traceback": None, # No Python traceback available in this case
862 | },
863 | "stdout": "", # Default values on timeout
864 | "stderr": "",
865 | "result": None,
866 | "elapsed": 0, # No Python elapsed time available
867 | "wall_ms": timeout_ms, # Wall time is the timeout duration
868 | }
869 | except Exception as e:
870 | # Catch potential Playwright communication errors during evaluate/wait
871 | logger.error(
872 | f"Error communicating during execution (id: {exec_id}): {e}", exc_info=True
873 | )
874 | # Return a structured communication error
875 | return {
876 | "ok": False,
877 | "error": {
878 | "type": "CommunicationError",
879 | "message": f"Error communicating with sandbox during execution: {e}",
880 | "traceback": None, # Or potentially include JS stack if available from e
881 | },
882 | "stdout": "",
883 | "stderr": "",
884 | "result": None,
885 | "elapsed": 0,
886 | "wall_ms": 0,
887 | }
888 | finally:
889 | # Always remove the message handler for this execution ID
890 | self._message_handlers.pop(exec_id, None)
891 |
892 | # --- Validate the structure of the received response ---
893 | if not isinstance(data, dict) or "ok" not in data:
894 | logger.error(
895 | f"Received malformed response from sandbox (id: {exec_id}, structure invalid): {str(data)[:500]}"
896 | )
897 | # Return a structured error indicating the malformed response
898 | return {
899 | "ok": False,
900 | "error": {
901 | "type": "MalformedResponseError",
902 | "message": "Received malformed or incomplete response from sandbox.",
903 | "traceback": None,
904 | # Safely include details for debugging, converting non-serializable types to string
905 | "details": data
906 | if isinstance(data, (dict, list, str, int, float, bool, type(None)))
907 | else str(data),
908 | },
909 | # Provide default values for other fields
910 | "stdout": "",
911 | "stderr": "",
912 | "result": None,
913 | "elapsed": 0,
914 | # Try to get wall_ms if data is a dict, otherwise default to 0
915 | "wall_ms": data.get("wall_ms", 0) if isinstance(data, dict) else 0,
916 | }
917 |
918 | # --- Ensure essential fields exist with default values before returning ---
919 | # This guarantees the caller receives a consistent structure even if the sandbox
920 | # somehow missed fields (though the JS side now also sets defaults).
921 | data.setdefault("stdout", "")
922 | data.setdefault("stderr", "")
923 | data.setdefault("result", None)
924 | data.setdefault("elapsed", 0)
925 | data.setdefault("wall_ms", 0)
926 | # Ensure 'error' field is present if 'ok' is false
927 | if not data.get("ok", False):
928 | data.setdefault(
929 | "error",
930 | {
931 | "type": "UnknownSandboxError",
932 | "message": "Sandbox reported failure with no specific details.",
933 | },
934 | )
935 | else:
936 | # Ensure 'error' is None if 'ok' is true
937 | data["error"] = None
938 |
939 | # Return the validated and defaulted data dictionary
940 | return data
941 |
942 | async def reset_repl_state(self) -> Dict[str, Any]:
943 | """Sends a reset request to the REPL sandbox."""
944 | if not PLAYWRIGHT_AVAILABLE:
945 | return {
946 | "ok": False,
947 | "error": {"type": "SetupError", "message": "Playwright not installed."},
948 | }
949 | if not self.page or self.page.is_closed():
950 | return {"ok": False, "error": {"type": "StateError", "message": "REPL page is closed."}}
951 | if not self.ready_evt.is_set():
952 | return {
953 | "ok": False,
954 | "error": {"type": "StateError", "message": "REPL sandbox is not ready."},
955 | }
956 |
957 | reset_id = f"reset-{uuid.uuid4().hex[:8]}"
958 | response_queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue()
959 | self._message_handlers[reset_id] = response_queue
960 | try:
961 | message_payload = {"type": "reset", "id": reset_id}
962 | logger.debug(f"Sending REPL reset message (id: {reset_id})")
963 | await self.page.evaluate("window.postMessage", message_payload)
964 | logger.debug(f"Waiting for REPL reset confirmation (id: {reset_id}, timeout: 5s)...")
965 | data = await asyncio.wait_for(response_queue.get(), timeout=5.0)
966 | logger.debug(f"Received REPL reset confirmation: {data}")
967 | return data
968 | except asyncio.TimeoutError:
969 | logger.warning(f"Timeout waiting for REPL reset confirmation (id: {reset_id})")
970 | return {
971 | "ok": False,
972 | "error": {
973 | "type": "TimeoutError",
974 | "message": "Timeout waiting for reset confirmation.",
975 | },
976 | }
977 | except Exception as e:
978 | logger.error(f"Error during REPL reset call (id: {reset_id}): {e}", exc_info=True)
979 | return {
980 | "ok": False,
981 | "error": {
982 | "type": "CommunicationError",
983 | "message": f"Error during reset operation: {e}",
984 | },
985 | }
986 | finally:
987 | self._message_handlers.pop(reset_id, None)
988 |
989 | async def _inject_mcpfs_stub(self) -> None:
990 | """Creates a minimal stub module `mcpfs` inside the Pyodide interpreter."""
991 | if not PLAYWRIGHT_AVAILABLE:
992 | logger.warning("Playwright not available, cannot inject mcpfs stub.")
993 | return
994 | # This stub code is executed within Pyodide, it doesn't need COMMON_PACKAGES_JSON from host Python
995 | stub_code = r"""
996 | import sys
997 | import types
998 | import asyncio
999 | import json
1000 | from js import globalThis
1001 |
1002 | # Simple check if stub already exists
1003 | if "mcpfs" in sys.modules:
1004 | print("mcpfs module stub already exists.")
1005 | else:
1006 | print("Initializing mcpfs module stub...")
1007 | _mcpfs_msg_id_counter = 0
1008 | _mcpfs_pending_futures = {}
1009 |
1010 | async def _mcpfs_roundtrip(op: str, *args):
1011 | '''Sends an operation to the host and waits for the response.'''
1012 | nonlocal _mcpfs_msg_id_counter
1013 | _mcpfs_msg_id_counter += 1
1014 | current_id = f"mcpfs-{_mcpfs_msg_id_counter}"
1015 |
1016 | loop = asyncio.get_running_loop()
1017 | fut = loop.create_future()
1018 | _mcpfs_pending_futures[current_id] = fut
1019 |
1020 | payload = {"type": "mcpfs", "id": current_id, "op": op, "args": args}
1021 | globalThis.postMessage(payload)
1022 |
1023 | try:
1024 | response = await asyncio.wait_for(fut, timeout=15.0)
1025 | except asyncio.TimeoutError:
1026 | raise RuntimeError(f"Timeout waiting for host mcpfs op '{op}' (id: {current_id})")
1027 | finally:
1028 | _mcpfs_pending_futures.pop(current_id, None)
1029 |
1030 | if response is None: raise RuntimeError(f"Null response from host for mcpfs op '{op}' (id: {current_id})")
1031 | if "error" in response:
1032 | err_details = response.get('details', '')
1033 | raise RuntimeError(f"Host error for mcpfs op '{op}': {response['error']} {err_details}")
1034 | return response.get("result")
1035 |
1036 | def _mcpfs_message_callback(event):
1037 | '''Callback attached to Pyodide's message listener to resolve futures.'''
1038 | data = event.data
1039 | if isinstance(data, dict) and data.get("type") == "mcpfs_response":
1040 | msg_id = data.get("id")
1041 | fut = _mcpfs_pending_futures.get(msg_id)
1042 | if fut and not fut.done(): fut.set_result(data)
1043 |
1044 | globalThis.addEventListener("message", _mcpfs_message_callback)
1045 |
1046 | mcpfs_module = types.ModuleType("mcpfs")
1047 | async def read_text_async(p): return await _mcpfs_roundtrip("read", p)
1048 | async def write_text_async(p, t): return await _mcpfs_roundtrip("write", p, t)
1049 | async def listdir_async(p): return await _mcpfs_roundtrip("list", p)
1050 | mcpfs_module.read_text_async = read_text_async
1051 | mcpfs_module.write_text_async = write_text_async
1052 | mcpfs_module.listdir_async = listdir_async
1053 | mcpfs_module.read_text = read_text_async
1054 | mcpfs_module.write_text = write_text_async
1055 | mcpfs_module.listdir = listdir_async
1056 | sys.modules["mcpfs"] = mcpfs_module
1057 | print("mcpfs Python module stub initialized successfully.")
1058 | # --- End of MCPFS Stub Logic ---
1059 | """
1060 | if not self.page or self.page.is_closed():
1061 | logger.error("Cannot inject mcpfs stub: Sandbox page is closed.")
1062 | return
1063 | try:
1064 | logger.debug("Injecting mcpfs stub into Pyodide environment...")
1065 | await self.page.evaluate(
1066 | f"""(async () => {{
1067 | try {{
1068 | if (typeof self.pyodide === 'undefined' || !self.pyodide.runPythonAsync) {{
1069 | console.error('Pyodide instance not ready for mcpfs stub injection.'); return;
1070 | }}
1071 | await self.pyodide.runPythonAsync(`{stub_code}`);
1072 | console.log("mcpfs Python stub injection script executed.");
1073 | }} catch (err) {{
1074 | console.error("Error running mcpfs stub injection Python code:", err);
1075 | globalThis.postMessage({{ type: 'error', id:'mcpfs_stub_inject_fail', error: {{ type: 'InjectionError', message: 'Failed to inject mcpfs stub: ' + err.toString() }} }}, "*");
1076 | }}
1077 | }})();"""
1078 | )
1079 | logger.info("mcpfs stub injection command sent to sandbox.")
1080 | except Exception as e:
1081 | logger.error(f"Failed to evaluate mcpfs stub injection script: {e}", exc_info=True)
1082 | # Don't raise ToolError here, log it. FS might not be critical.
1083 |
1084 |
1085 | # --- End of PyodideSandbox Class ---
1086 |
1087 |
1088 | ###############################################################################
1089 | # Browser / sandbox lifecycle helpers – with LRU eviction
1090 | ###############################################################################
1091 | async def _get_browser() -> "Browser": # type: ignore # Use string literal hint
1092 | """Initializes and returns the shared Playwright browser instance."""
1093 | global _BROWSER
1094 | if not PLAYWRIGHT_AVAILABLE:
1095 | raise RuntimeError("Playwright is not installed.")
1096 | browser_connected = False
1097 | if _BROWSER is not None:
1098 | try:
1099 | browser_connected = _BROWSER.is_connected()
1100 | except Exception as check_err:
1101 | logger.warning(
1102 | f"Error checking browser connection status: {check_err}. Assuming disconnected."
1103 | )
1104 | browser_connected = False
1105 | _BROWSER = None
1106 | if _BROWSER is None or not browser_connected:
1107 | logger.info("Launching headless Chromium for Pyodide sandbox...")
1108 | try:
1109 | playwright = await pw.async_playwright().start()
1110 | launch_options = {
1111 | "headless": True,
1112 | "args": [
1113 | "--no-sandbox",
1114 | "--disable-gpu",
1115 | "--disable-dev-shm-usage",
1116 | "--disable-features=Translate",
1117 | "--disable-extensions",
1118 | "--disable-component-extensions-with-background-pages",
1119 | "--disable-background-networking",
1120 | "--disable-sync",
1121 | "--metrics-recording-only",
1122 | "--disable-default-apps",
1123 | "--mute-audio",
1124 | "--no-first-run",
1125 | "--safebrowsing-disable-auto-update",
1126 | "--disable-popup-blocking",
1127 | "--disable-setuid-sandbox",
1128 | "--disable-web-security",
1129 | "--allow-file-access-from-files",
1130 | "--allow-universal-access-from-file-urls",
1131 | "--disable-permissions-api",
1132 | ],
1133 | "timeout": 90000,
1134 | }
1135 | _BROWSER = await playwright.chromium.launch(**launch_options)
1136 |
1137 | def _sync_cleanup():
1138 | global _BROWSER
1139 | if _BROWSER and _BROWSER.is_connected():
1140 | logger.info("Closing Playwright browser via atexit handler...")
1141 | try:
1142 | loop = asyncio.get_event_loop_policy().get_event_loop()
1143 | if loop.is_running():
1144 | future = asyncio.run_coroutine_threadsafe(_BROWSER.close(), loop)
1145 | future.result(timeout=15)
1146 | else:
1147 | loop.run_until_complete(_BROWSER.close())
1148 | logger.info("Playwright browser closed successfully via atexit.")
1149 | _BROWSER = None
1150 | except Exception as e:
1151 | logger.error(
1152 | f"Error during atexit Playwright browser cleanup: {e}", exc_info=True
1153 | )
1154 |
1155 | atexit.register(_sync_cleanup)
1156 | logger.info("Headless Chromium launched successfully and atexit cleanup registered.")
1157 | except Exception as e:
1158 | logger.error(f"Failed to launch Playwright browser: {e}", exc_info=True)
1159 | _BROWSER = None
1160 | raise ProviderError(f"Failed to launch browser for sandbox: {e}") from e
1161 | if not _BROWSER:
1162 | raise ProviderError("Browser instance is None after launch attempt.")
1163 | return _BROWSER
1164 |
1165 |
1166 | async def _get_sandbox(session_id: str, **kwargs) -> PyodideSandbox:
1167 | """Retrieves or creates a PyodideSandbox instance, managing LRU cache."""
1168 | global _GLOBAL_SEM, _PAGES
1169 | if not PLAYWRIGHT_AVAILABLE:
1170 | # Check upfront if Playwright is available
1171 | raise RuntimeError("Playwright is not installed. Cannot create sandboxes.")
1172 |
1173 | # Initialize the global semaphore if this is the first call
1174 | if _GLOBAL_SEM is None:
1175 | _GLOBAL_SEM = asyncio.Semaphore(GLOBAL_CONCURRENCY)
1176 | logger.debug(
1177 | f"Initialized global execution semaphore with concurrency {GLOBAL_CONCURRENCY}"
1178 | )
1179 |
1180 | # Check if a sandbox for this session ID already exists in our cache
1181 | sb = _PAGES.get(session_id)
1182 | if sb is not None:
1183 | page_valid = False
1184 | if sb.page:
1185 | # Verify the associated Playwright Page object is still open
1186 | try:
1187 | page_valid = not sb.page.is_closed()
1188 | except Exception as page_check_err:
1189 | # Handle potential errors during the check (e.g., context destroyed)
1190 | logger.warning(
1191 | f"Error checking page status for {session_id}: {page_check_err}. Assuming invalid."
1192 | )
1193 | page_valid = False
1194 |
1195 | if page_valid:
1196 | # If the page is valid, reuse the existing sandbox
1197 | logger.debug(f"Reusing existing sandbox session: {session_id}")
1198 | # Move the accessed sandbox to the end of the OrderedDict (marks it as recently used)
1199 | _PAGES.move_to_end(session_id)
1200 | sb.last_used = time.time() # Update last used timestamp
1201 | return sb
1202 | else:
1203 | # If the page is closed or invalid, remove the entry from the cache
1204 | logger.warning(f"Removing closed/invalid sandbox session from cache: {session_id}")
1205 | _PAGES.pop(session_id, None)
1206 | # Attempt to gracefully close the page object if it exists
1207 | if sb.page:
1208 | await sb._try_close_page("Invalid page found in cache")
1209 |
1210 | # If no valid sandbox found, create a new one, potentially evicting the LRU
1211 | while len(_PAGES) >= MAX_SANDBOXES:
1212 | # Remove the least recently used sandbox (first item in OrderedDict)
1213 | try:
1214 | victim_id, victim_sb = _PAGES.popitem(last=False)
1215 | logger.info(
1216 | f"Sandbox cache full ({len(_PAGES) + 1}/{MAX_SANDBOXES}). Evicting LRU session: {victim_id} "
1217 | f"(created: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(victim_sb.created_at))}, "
1218 | f"last used: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(victim_sb.last_used))})"
1219 | )
1220 | # Attempt to close the evicted sandbox's page
1221 | await victim_sb._try_close_page(f"LRU eviction (victim: {victim_id})")
1222 | except KeyError:
1223 | # Should not happen if len(_PAGES) >= MAX_SANDBOXES, but handle defensively
1224 | logger.warning("LRU eviction attempted but cache was empty.")
1225 | break # Avoid infinite loop if something is wrong
1226 |
1227 | logger.info(f"Creating new sandbox session: {session_id}")
1228 | browser = await _get_browser() # Get or initialize the shared browser instance
1229 | page: Optional["Page"] = None # type: ignore # Initialize page variable
1230 |
1231 | try:
1232 | # Create a new browser page (tab)
1233 | page = await browser.new_page()
1234 | # Set up logging and error handlers for this specific page
1235 | _wire_page_logging(page, session_id)
1236 | logger.debug(f"New browser page created for session {session_id}")
1237 |
1238 | # Create the PyodideSandbox object instance
1239 | sb = PyodideSandbox(
1240 | page=page, **kwargs
1241 | ) # Pass through any extra kwargs (like allow_network)
1242 |
1243 | # Initialize the sandbox (loads HTML, waits for ready signal)
1244 | await sb.init()
1245 |
1246 | # Add the newly created and initialized sandbox to the cache
1247 | _PAGES[session_id] = sb
1248 | logger.info(f"New sandbox session {session_id} created and initialized successfully.")
1249 | return sb
1250 |
1251 | except Exception as e:
1252 | # Handle errors during page creation or sandbox initialization
1253 | logger.error(f"Failed to create or initialize new sandbox {session_id}: {e}", exc_info=True)
1254 | # If the page was created but initialization failed, try to close it
1255 | if page and not page.is_closed():
1256 | # Use a temporary sandbox object just to call the closing method
1257 | await PyodideSandbox(page=page)._try_close_page(
1258 | f"Failed sandbox creation/init ({session_id})"
1259 | )
1260 | # Re-raise the exception, preserving type if it's a known error type
1261 | if isinstance(e, (ToolError, ProviderError)):
1262 | raise e
1263 | # Wrap unexpected errors in ProviderError for consistent error handling upstream
1264 | raise ProviderError(f"Failed to create sandbox {session_id}: {e}") from e
1265 |
1266 |
1267 | async def _close_all_sandboxes():
1268 | """Gracefully close all active sandbox pages and the browser."""
1269 | global _BROWSER, _PAGES
1270 | logger.info("Closing all active Pyodide sandboxes...")
1271 | page_close_tasks = []
1272 | sandboxes_to_close = list(_PAGES.values())
1273 | _PAGES.clear()
1274 | for sb in sandboxes_to_close:
1275 | close_task = asyncio.create_task(sb._try_close_page("Global shutdown"))
1276 | page_close_tasks.append(close_task)
1277 | if page_close_tasks:
1278 | gathered_results = await asyncio.gather(*page_close_tasks, return_exceptions=True)
1279 | closed_count = sum(1 for result in gathered_results if not isinstance(result, Exception))
1280 | errors = [result for result in gathered_results if isinstance(result, Exception)]
1281 | logger.info(
1282 | f"Attempted to close {len(page_close_tasks)} sandbox pages. Success: {closed_count}."
1283 | )
1284 | if errors:
1285 | logger.warning(f"{len(errors)} errors during page close: {errors}")
1286 |
1287 | browser_needs_closing = False
1288 | if _BROWSER:
1289 | try:
1290 | browser_needs_closing = _BROWSER.is_connected()
1291 | except Exception as browser_check_err:
1292 | logger.warning(
1293 | f"Error checking browser connection during close: {browser_check_err}. Assuming needs closing."
1294 | )
1295 | browser_needs_closing = True
1296 | if browser_needs_closing:
1297 | logger.info("Closing Playwright browser instance...")
1298 | try:
1299 | await _BROWSER.close()
1300 | logger.info("Playwright browser closed successfully.")
1301 | except Exception as e:
1302 | logger.error(f"Error closing Playwright browser: {e}")
1303 | _BROWSER = None
1304 |
1305 |
1306 |
1307 | def display_sandbox_result(
1308 | title: str, result: Optional[Dict[str, Any]], code_str: Optional[str] = None
1309 | ) -> None:
1310 | """Display sandbox execution result with enhanced formatting."""
1311 | console.print(Rule(f"[bold cyan]{escape(title)}[/bold cyan]"))
1312 |
1313 | if code_str:
1314 | console.print(
1315 | Panel(
1316 | Syntax(
1317 | code_str.strip(), "python", theme="monokai", line_numbers=True, word_wrap=True
1318 | ),
1319 | title="Executed Code",
1320 | border_style="blue",
1321 | padding=(1, 2),
1322 | )
1323 | )
1324 |
1325 | if result is None:
1326 | console.print(
1327 | Panel(
1328 | "[bold yellow]No result object returned from tool call.[/]",
1329 | title="Warning",
1330 | border_style="yellow",
1331 | )
1332 | )
1333 | console.print()
1334 | return
1335 |
1336 | # Check for errors based on the result structure from safe_tool_call
1337 | if not result.get("success", False) and "error" in result:
1338 | error_msg = result.get("error", "Unknown error")
1339 | error_type = result.get("error_type", "UnknownError")
1340 | error_code = result.get("error_code", "UNKNOWN")
1341 | details = result.get("details", {})
1342 |
1343 | error_renderable = f"[bold red]:x: Operation Failed ({escape(error_type)} / {escape(error_code)}):[/]\n{escape(error_msg)}"
1344 | if details:
1345 | try:
1346 | details_str = escape(str(details)) # Basic string representation
1347 | error_renderable += f"\n\n[bold]Details:[/]\n{details_str}"
1348 | except Exception:
1349 | error_renderable += "\n\n[bold]Details:[/]\n(Could not display details)"
1350 |
1351 | console.print(
1352 | Panel(error_renderable, title="Error", border_style="red", padding=(1, 2), expand=False)
1353 | )
1354 | console.print()
1355 | return
1356 |
1357 | # --- Display Success Case ---
1358 | actual_result = result.get(
1359 | "result", {}
1360 | ) # Get the nested result dict from execute_python/repl_python
1361 |
1362 | # Create output panel for stdout/stderr
1363 | output_parts = []
1364 | if stdout := actual_result.get("stdout", ""):
1365 | output_parts.append(f"[bold green]STDOUT:[/]\n{escape(stdout)}")
1366 |
1367 | if stderr := actual_result.get("stderr", ""):
1368 | if output_parts:
1369 | output_parts.append("\n" + ("-" * 20) + "\n") # Separator
1370 | output_parts.append(f"[bold red]STDERR:[/]\n{escape(stderr)}")
1371 |
1372 | if output_parts:
1373 | console.print(
1374 | Panel(
1375 | "\n".join(output_parts),
1376 | title="Output (stdout/stderr)",
1377 | border_style="yellow",
1378 | padding=(1, 2),
1379 | )
1380 | )
1381 | else:
1382 | console.print("[dim]No stdout or stderr captured.[/dim]")
1383 |
1384 | # Display result value if present and not None
1385 | result_value = actual_result.get(
1386 | "result"
1387 | ) # This is the value assigned to 'result' in the executed code
1388 | if result_value is not None:
1389 | try:
1390 | # Attempt to pretty-print common types
1391 | if isinstance(result_value, (dict, list)):
1392 | result_str = str(result_value) # Keep it simple for now
1393 | else:
1394 | result_str = str(result_value)
1395 |
1396 | # Limit length for display
1397 | max_len = 500
1398 | display_str = result_str[:max_len] + ("..." if len(result_str) > max_len else "")
1399 |
1400 | console.print(
1401 | Panel(
1402 | Syntax(
1403 | display_str, "python", theme="monokai", line_numbers=False, word_wrap=True
1404 | ),
1405 | title="Result Variable ('result')",
1406 | border_style="green",
1407 | padding=(1, 2),
1408 | )
1409 | )
1410 | except Exception as e:
1411 | console.print(
1412 | Panel(
1413 | f"[yellow]Could not format result value: {e}[/]",
1414 | title="Result Variable ('result')",
1415 | border_style="yellow",
1416 | )
1417 | )
1418 | console.print(f"Raw Result Type: {type(result_value)}")
1419 | try:
1420 | console.print(f"Raw Result Repr: {escape(repr(result_value)[:500])}...")
1421 | except Exception:
1422 | pass
1423 |
1424 | # Display execution stats
1425 | stats_table = Table(
1426 | title="Execution Statistics",
1427 | box=box.ROUNDED,
1428 | show_header=False,
1429 | padding=(0, 1),
1430 | border_style="dim",
1431 | )
1432 | stats_table.add_column("Metric", style="cyan", justify="right")
1433 | stats_table.add_column("Value", style="white")
1434 |
1435 | if "elapsed_py_ms" in actual_result:
1436 | stats_table.add_row("Python Execution Time", f"{actual_result['elapsed_py_ms']:.2f} ms")
1437 | if "elapsed_wall_ms" in actual_result:
1438 | stats_table.add_row("Sandbox Wall Clock Time", f"{actual_result['elapsed_wall_ms']:.2f} ms")
1439 | if "total_duration_ms" in result: # From safe_tool_call wrapper
1440 | stats_table.add_row("Total Tool Call Time", f"{result['total_duration_ms']:.2f} ms")
1441 | if "session_id" in actual_result:
1442 | stats_table.add_row("Session ID", actual_result["session_id"])
1443 | if "handle" in actual_result:
1444 | stats_table.add_row("REPL Handle", actual_result["handle"])
1445 |
1446 | if stats_table.row_count > 0:
1447 | console.print(stats_table)
1448 |
1449 | console.print() # Add spacing
1450 |
1451 |
1452 | ###############################################################################
1453 | # mcpfs bridge – listens for postMessage & proxies to secure FS tool
1454 | ###############################################################################
1455 | async def _listen_for_mcpfs_calls(page: "Page"): # type: ignore # Use string literal hint
1456 | """Sets up listener for 'mcpfs' messages from the sandbox page."""
1457 | if not PLAYWRIGHT_AVAILABLE:
1458 | logger.warning("Playwright not available, cannot listen for mcpfs calls.")
1459 | return
1460 |
1461 | async def _handle_mcpfs_message(payload: Any):
1462 | """Processes 'mcpfs' request from Pyodide and sends 'mcpfs_response' back."""
1463 | data = payload
1464 | is_mcpfs_message = isinstance(data, dict) and data.get("type") == "mcpfs"
1465 | if not is_mcpfs_message:
1466 | return
1467 |
1468 | call_id = data.get("id")
1469 | op = data.get("op")
1470 | args = data.get("args", [])
1471 | if not call_id or not op:
1472 | logger.warning(
1473 | f"MCPFS Bridge: Received invalid mcpfs message (missing id or op): {data}"
1474 | )
1475 | return
1476 | response_payload: dict[str, Any] = {"type": "mcpfs_response", "id": call_id}
1477 | try:
1478 | try:
1479 | from ultimate_mcp_server.tools import filesystem as fs
1480 | except ImportError as e:
1481 | logger.error("MCPFS Bridge: Failed to import 'filesystem' tool.", exc_info=True)
1482 | raise ToolError("Filesystem tool backend not available.") from e
1483 | if _VERBOSE_SANDBOX_LOGGING > 1:
1484 | logger.debug(f"MCPFS Bridge: Received op='{op}', args={args}, id={call_id}")
1485 |
1486 | if op == "read":
1487 | if len(args) != 1 or not isinstance(args[0], str):
1488 | raise ValueError("read requires 1 string arg (path)")
1489 | res = await fs.read_file(path=args[0])
1490 | if res.get("success") and isinstance(res.get("content"), list) and res["content"]:
1491 | file_content = res["content"][0].get("text")
1492 | if file_content is None:
1493 | raise ToolError("Read succeeded but missing 'text' key.")
1494 | response_payload["result"] = file_content
1495 | else:
1496 | raise ToolError(res.get("error", "Read failed"), details=res.get("details"))
1497 | elif op == "write":
1498 | if len(args) != 2 or not isinstance(args[0], str) or not isinstance(args[1], str):
1499 | raise ValueError("write requires 2 string args (path, content)")
1500 | res = await fs.write_file(path=args[0], content=args[1])
1501 | if res.get("success"):
1502 | response_payload["result"] = True
1503 | else:
1504 | raise ToolError(res.get("error", "Write failed"), details=res.get("details"))
1505 | elif op == "list":
1506 | if len(args) != 1 or not isinstance(args[0], str):
1507 | raise ValueError("list requires 1 string arg (path)")
1508 | res = await fs.list_directory(path=args[0])
1509 | if res.get("success"):
1510 | response_payload["result"] = res.get("entries", [])
1511 | else:
1512 | raise ToolError(res.get("error", "List failed"), details=res.get("details"))
1513 | else:
1514 | raise ValueError(f"Unsupported mcpfs operation: '{op}'")
1515 | except (ToolError, ToolInputError, ProviderError, ValueError) as tool_exc:
1516 | error_message = f"{type(tool_exc).__name__}: {tool_exc}"
1517 | logger.warning(
1518 | f"MCPFS Bridge Error processing op='{op}' (id={call_id}): {error_message}"
1519 | )
1520 | response_payload["error"] = error_message
1521 | if hasattr(tool_exc, "details") and tool_exc.details:
1522 | try:
1523 | response_payload["details"] = json.loads(
1524 | json.dumps(tool_exc.details, default=str)
1525 | )
1526 | except Exception:
1527 | response_payload["details"] = {"error": "Serialization failed"}
1528 | except Exception as exc:
1529 | error_message = f"Unexpected Host Error: {exc}"
1530 | logger.error(
1531 | f"Unexpected MCPFS Bridge Error (op='{op}', id={call_id}): {error_message}",
1532 | exc_info=True,
1533 | )
1534 | response_payload["error"] = error_message
1535 | try:
1536 | response_successful = "error" not in response_payload
1537 | if _VERBOSE_SANDBOX_LOGGING > 1:
1538 | logger.debug(
1539 | f"MCPFS Bridge: Sending response (op='{op}', id={call_id}, success={response_successful})"
1540 | )
1541 | await page.evaluate(JS_POST_MESSAGE, response_payload)
1542 | except Exception as post_err:
1543 | logger.warning(
1544 | f"Failed to send mcpfs response back to sandbox (id: {call_id}, op: '{op}'): {post_err}"
1545 | )
1546 |
1547 | handler_func_name = "_handleMcpFsMessageFromHost"
1548 | try:
1549 | await page.expose_function(handler_func_name, _handle_mcpfs_message)
1550 | await page.evaluate(f"""
1551 | if (!window._mcpfsListenerAttached) {{
1552 | console.log('Setting up MCPFS message listener in browser context...');
1553 | window.addEventListener('message', (event) => {{
1554 | if (event.data && event.data.type === 'mcpfs' && typeof window.{handler_func_name} === 'function') {{
1555 | window.{handler_func_name}(event.data);
1556 | }}
1557 | }});
1558 | window._mcpfsListenerAttached = true;
1559 | console.log('MCPFS message listener attached.');
1560 | }}
1561 | """)
1562 | logger.info("MCPFS listener bridge established successfully.")
1563 | except Exception as e:
1564 | logger.error(f"Failed to set up MCPFS listener bridge: {e}", exc_info=True)
1565 | raise ToolError(f"Filesystem bridge listener setup failed: {e}") from e
1566 |
1567 |
1568 | def _format_sandbox_error(error_payload: Optional[Dict[str, Any]]) -> str:
1569 | if not error_payload or not isinstance(error_payload, dict):
1570 | return "Unknown sandbox execution error."
1571 | err_type = error_payload.get("type", "UnknownError")
1572 | err_msg = error_payload.get("message", "No details provided.")
1573 | # Optionally include traceback snippet if needed, but keep main message clean
1574 | tb = error_payload.get("traceback")
1575 | if tb:
1576 | err_msg += f"\nTraceback (see logs/details):\n{str(tb)[:200]}..."
1577 | return f"{err_type} - {err_msg}"
1578 |
1579 |
1580 | ###############################################################################
1581 | # Standalone Tool Functions (execute_python, repl_python)
1582 | ###############################################################################
1583 | @with_tool_metrics
1584 | @with_error_handling
1585 | async def execute_python(
1586 | code: str,
1587 | packages: Optional[List[str]] = None,
1588 | wheels: Optional[List[str]] = None,
1589 | allow_network: bool = False,
1590 | allow_fs: bool = False,
1591 | session_id: Optional[str] = None,
1592 | timeout_ms: int = 15_000,
1593 | ctx: Optional[Dict[str, Any]] = None, # Context often used by decorators
1594 | ) -> Dict[str, Any]:
1595 | """
1596 | Runs Python code in a one-shot Pyodide sandbox.
1597 |
1598 | Args:
1599 | code: The Python code string to execute.
1600 | packages: A list of Pyodide packages to ensure are loaded. Do not include stdlib modules.
1601 | wheels: A list of Python wheel URLs to install via micropip.
1602 | allow_network: If True, allows network access (e.g., for micropip to PyPI).
1603 | allow_fs: If True, enables the mcpfs filesystem bridge (requires host setup).
1604 | session_id: Optional ID to reuse or create a specific sandbox session. If None, a new ID is generated.
1605 | timeout_ms: Timeout for waiting for the sandbox execution result (in milliseconds).
1606 | ctx: Optional context dictionary, often passed by framework/decorators.
1607 |
1608 | Returns:
1609 | A dictionary containing execution results:
1610 | {
1611 | 'success': bool,
1612 | 'stdout': str,
1613 | 'stderr': str,
1614 | 'result': Any, # Value of the 'result' variable in the Python code, if set
1615 | 'elapsed_py_ms': int, # Time spent executing Python code (reported by sandbox)
1616 | 'elapsed_wall_ms': int, # Total wall clock time from JS perspective (reported by sandbox)
1617 | 'session_id': str,
1618 | 'error_message': Optional[str], # Formatted error if success is False
1619 | 'error_details': Optional[Dict], # Original error dict from sandbox if success is False
1620 | }
1621 |
1622 | Raises:
1623 | ProviderError: If the sandbox environment (Playwright/browser) cannot be set up.
1624 | ToolInputError: If input arguments are invalid.
1625 | ToolError: If sandbox execution fails (contains formatted message and details).
1626 | """
1627 | if not PLAYWRIGHT_AVAILABLE:
1628 | raise ProviderError("Playwright dependency is missing for Python Sandbox.")
1629 | if not isinstance(code, str) or not code:
1630 | raise ToolInputError(
1631 | "Input 'code' must be a non-empty string.", param="code", value=repr(code)
1632 | )
1633 | if not isinstance(timeout_ms, int) or timeout_ms <= 0:
1634 | raise ToolInputError(
1635 | "Input 'timeout_ms' must be a positive integer.", param="timeout_ms", value=timeout_ms
1636 | )
1637 | # Basic type checks for lists/bools - could add more specific validation
1638 | if packages is not None and not isinstance(packages, list):
1639 | raise ToolInputError(
1640 | "Input 'packages' must be a list or None.", param="packages", value=packages
1641 | )
1642 | if wheels is not None and not isinstance(wheels, list):
1643 | raise ToolInputError("Input 'wheels' must be a list or None.", param="wheels", value=wheels)
1644 | if not isinstance(allow_network, bool):
1645 | raise ToolInputError(
1646 | "Input 'allow_network' must be a boolean.", param="allow_network", value=allow_network
1647 | )
1648 | if not isinstance(allow_fs, bool):
1649 | raise ToolInputError(
1650 | "Input 'allow_fs' must be a boolean.", param="allow_fs", value=allow_fs
1651 | )
1652 | if session_id is not None and not isinstance(session_id, str):
1653 | raise ToolInputError(
1654 | "Input 'session_id' must be a string or None.", param="session_id", value=session_id
1655 | )
1656 |
1657 | # Normalize package/wheel lists
1658 | # IMPORTANT: Filter out common stdlib modules that shouldn't be passed
1659 | stdlib_modules_to_filter = {
1660 | "math",
1661 | "sys",
1662 | "os",
1663 | "json",
1664 | "io",
1665 | "contextlib",
1666 | "time",
1667 | "base64",
1668 | "traceback",
1669 | "collections",
1670 | "re",
1671 | "datetime",
1672 | }
1673 | packages_normalized = [pkg for pkg in (packages or []) if pkg not in stdlib_modules_to_filter]
1674 | wheels_normalized = wheels or []
1675 |
1676 | # Generate a session ID if one wasn't provided
1677 | current_session_id = session_id or f"exec-{uuid.uuid4().hex[:12]}" # Add prefix for clarity
1678 |
1679 | # Get or create the sandbox instance
1680 | try:
1681 | # Assuming _get_sandbox is defined elsewhere and returns PyodideSandbox instance
1682 | sb = await _get_sandbox(current_session_id, allow_network=allow_network, allow_fs=allow_fs)
1683 | except Exception as e:
1684 | # Catch potential errors during sandbox acquisition/initialization
1685 | if isinstance(e, (ToolError, ProviderError)):
1686 | raise e # Re-raise known error types
1687 | # Wrap unexpected errors
1688 | raise ProviderError(
1689 | f"Failed to get or initialize sandbox '{current_session_id}': {e}",
1690 | tool_name="python_sandbox",
1691 | cause=e,
1692 | ) from e
1693 |
1694 | t0 = time.perf_counter() # Start timer just before execute call
1695 | data: Dict[str, Any] = {} # Initialize data dict
1696 |
1697 | # Execute the code within the sandbox
1698 | try:
1699 | # Call the execute method on the sandbox object
1700 | # Pass repl_mode=False for one-shot execution
1701 | data = await sb.execute(
1702 | code, packages_normalized, wheels_normalized, timeout_ms, repl_mode=False
1703 | )
1704 | except Exception as e:
1705 | # Catch potential host-side errors during the .execute() call itself
1706 | # (e.g., Playwright communication errors not caught internally by execute)
1707 | wall_ms = int((time.perf_counter() - t0) * 1000)
1708 | logger.error(
1709 | f"Unexpected host error calling sandbox execute for {current_session_id}: {e}",
1710 | exc_info=True,
1711 | )
1712 | raise ToolError(
1713 | f"Unexpected host error during sandbox execution call: {e}",
1714 | error_code="HostExecutionError",
1715 | details={"session_id": current_session_id, "elapsed_wall_ms": wall_ms, "cause": str(e)},
1716 | ) from e
1717 |
1718 | # Process the results received from the sandbox
1719 | wall_ms_host = int((time.perf_counter() - t0) * 1000) # Wall time measured by host
1720 | is_success = data.get("ok", False)
1721 | error_info = data.get(
1722 | "error"
1723 | ) # This is the structured {type, message, traceback} dict from sandbox
1724 | js_wall_ms = int(data.get("wall_ms", 0)) # Wall time reported by JS sandbox handler
1725 |
1726 | # Format error message IF execution failed inside the sandbox
1727 | error_message_for_caller = None
1728 | error_code_for_caller = "UnknownSandboxError" # Default error code
1729 | if not is_success:
1730 | error_message_for_caller = _format_sandbox_error(
1731 | error_info
1732 | ) # Use helper to get "Type - Message" string
1733 | if isinstance(error_info, dict):
1734 | error_code_for_caller = error_info.get(
1735 | "type", "UnknownSandboxError"
1736 | ) # Get specific code
1737 |
1738 | # Prepare structured logging details
1739 | log_details = {
1740 | "session_id": current_session_id,
1741 | "elapsed_wall_ms_host": wall_ms_host,
1742 | "elapsed_wall_ms_js": js_wall_ms, # Log both wall times for comparison
1743 | "elapsed_py_ms": int(data.get("elapsed", 0)),
1744 | "packages_requested": packages or [], # Log original requested packages
1745 | "packages_loaded": packages_normalized, # Log packages actually sent to loadPackage
1746 | "wheels_count": len(wheels_normalized),
1747 | "stdout_len": len(data.get("stdout", "")),
1748 | "stderr_len": len(data.get("stderr", "")),
1749 | "result_type": type(data.get("result")).__name__,
1750 | "success": is_success,
1751 | "repl_mode": False,
1752 | }
1753 |
1754 | # Log and return/raise based on success
1755 | if is_success:
1756 | logger.success(
1757 | f"Python code executed successfully (session: {current_session_id})",
1758 | TaskType.CODE_EXECUTION, # Assumes TaskType is defined/imported
1759 | **log_details,
1760 | )
1761 | # Return success dictionary matching specified structure
1762 | return {
1763 | "success": True,
1764 | "stdout": data.get("stdout", ""),
1765 | "stderr": data.get("stderr", ""),
1766 | "result": data.get("result"), # Can be None
1767 | "elapsed_py_ms": int(data.get("elapsed", 0)),
1768 | "elapsed_wall_ms": js_wall_ms or wall_ms_host, # Prefer JS wall time
1769 | "session_id": current_session_id,
1770 | "error_message": None, # Explicitly None on success
1771 | "error_details": None, # Explicitly None on success
1772 | }
1773 | else:
1774 | # Log the failure with details
1775 | logger.error(
1776 | f"Python code execution failed (session: {current_session_id}): {error_message_for_caller}",
1777 | TaskType.CODE_EXECUTION, # Assumes TaskType is defined/imported
1778 | **log_details,
1779 | error_details=error_info, # Log the original structured error details
1780 | )
1781 | # Raise a ToolError containing the formatted message and original details
1782 | raise ToolError(
1783 | f"Python execution failed: {error_message_for_caller}", # User-friendly message
1784 | error_code=error_code_for_caller, # Specific error code from sandbox
1785 | details=error_info, # Original structured error from sandbox
1786 | )
1787 |
1788 |
1789 | @with_tool_metrics
1790 | @with_error_handling
1791 | async def repl_python(
1792 | code: str,
1793 | packages: Optional[List[str]] = None,
1794 | wheels: Optional[List[str]] = None,
1795 | allow_network: bool = False,
1796 | allow_fs: bool = False,
1797 | handle: Optional[str] = None, # Session handle for persistence
1798 | timeout_ms: int = 15_000,
1799 | reset: bool = False, # Flag to reset the REPL state before execution
1800 | ctx: Optional[Dict[str, Any]] = None, # Context often used by decorators
1801 | ) -> Dict[str, Any]:
1802 | """
1803 | Runs Python code in a persistent REPL-like sandbox environment.
1804 |
1805 | Args:
1806 | code: The Python code string to execute in the session. Can be empty if only resetting.
1807 | packages: Additional Pyodide packages to ensure are loaded for this specific call.
1808 | wheels: Additional Python wheel URLs to install for this specific call.
1809 | allow_network: If True, allows network access for the sandbox session.
1810 | allow_fs: If True, enables the mcpfs filesystem bridge for the session.
1811 | handle: A specific session ID to use. If None, a new session is created.
1812 | Use the returned handle for subsequent calls to maintain state.
1813 | timeout_ms: Timeout for waiting for this specific execution call.
1814 | reset: If True, clears the REPL session's state (_MCP_REPL_NS) before executing code.
1815 | ctx: Optional context dictionary.
1816 |
1817 | Returns:
1818 | A dictionary containing execution results for *this call*:
1819 | {
1820 | 'success': bool, # Success of *this specific code execution* (or reset)
1821 | 'stdout': str,
1822 | 'stderr': str,
1823 | 'result': Any, # Value of 'result' variable from this execution, if set
1824 | 'elapsed_py_ms': int,
1825 | 'elapsed_wall_ms': int,
1826 | 'handle': str, # The session handle (same as input or newly generated)
1827 | 'error_message': Optional[str], # Formatted error if success is False
1828 | 'error_details': Optional[Dict], # Original error dict if success is False
1829 | 'reset_status': Optional[Dict], # Included only if reset=True, contains reset ack
1830 | }
1831 |
1832 | Raises:
1833 | ProviderError: If the sandbox environment cannot be set up.
1834 | ToolInputError: If input arguments are invalid.
1835 | ToolError: If a non-recoverable error occurs during execution (contains details).
1836 | Note: Standard Python errors within the code are returned in the 'error' fields,
1837 | not typically raised as ToolError unless they prevent result processing.
1838 | """
1839 | if not PLAYWRIGHT_AVAILABLE:
1840 | raise ProviderError("Playwright dependency is missing for Python Sandbox.")
1841 | # Code can be empty if reset is True
1842 | if not isinstance(code, str):
1843 | raise ToolInputError("Input 'code' must be a string.", param="code", value=repr(code))
1844 | if not code and not reset:
1845 | raise ToolInputError(
1846 | "Input 'code' cannot be empty unless 'reset' is True.", param="code", value=repr(code)
1847 | )
1848 | if not isinstance(timeout_ms, int) or timeout_ms <= 0:
1849 | raise ToolInputError(
1850 | "Input 'timeout_ms' must be a positive integer.", param="timeout_ms", value=timeout_ms
1851 | )
1852 | # Basic type checks - can be expanded
1853 | if packages is not None and not isinstance(packages, list):
1854 | raise ToolInputError(
1855 | "Input 'packages' must be a list or None.", param="packages", value=packages
1856 | )
1857 | if wheels is not None and not isinstance(wheels, list):
1858 | raise ToolInputError("Input 'wheels' must be a list or None.", param="wheels", value=wheels)
1859 | if not isinstance(allow_network, bool):
1860 | raise ToolInputError(
1861 | "Input 'allow_network' must be a boolean.", param="allow_network", value=allow_network
1862 | )
1863 | if not isinstance(allow_fs, bool):
1864 | raise ToolInputError(
1865 | "Input 'allow_fs' must be a boolean.", param="allow_fs", value=allow_fs
1866 | )
1867 | if handle is not None and not isinstance(handle, str):
1868 | raise ToolInputError(
1869 | "Input 'handle' must be a string or None.", param="handle", value=handle
1870 | )
1871 | if not isinstance(reset, bool):
1872 | raise ToolInputError("Input 'reset' must be a boolean.", param="reset", value=reset)
1873 |
1874 | # IMPORTANT: Filter out common stdlib modules that shouldn't be passed
1875 | stdlib_modules_to_filter = {
1876 | "math",
1877 | "sys",
1878 | "os",
1879 | "json",
1880 | "io",
1881 | "contextlib",
1882 | "time",
1883 | "base64",
1884 | "traceback",
1885 | "collections",
1886 | "re",
1887 | "datetime",
1888 | }
1889 | packages_normalized = [pkg for pkg in (packages or []) if pkg not in stdlib_modules_to_filter]
1890 | wheels_normalized = wheels or []
1891 |
1892 | # Use provided handle or generate a new persistent one
1893 | session_id = handle or f"repl-{uuid.uuid4().hex[:12]}"
1894 |
1895 | # Get or create the sandbox instance (will reuse if handle exists and page is open)
1896 | try:
1897 | # Pass allow_network/allow_fs, they are session-level properties
1898 | sb = await _get_sandbox(session_id, allow_network=allow_network, allow_fs=allow_fs)
1899 | except Exception as e:
1900 | if isinstance(e, (ToolError, ProviderError)):
1901 | raise e
1902 | raise ProviderError(
1903 | f"Failed to get or initialize REPL sandbox '{session_id}': {e}",
1904 | tool_name="python_sandbox",
1905 | cause=e,
1906 | ) from e
1907 |
1908 | t0 = time.perf_counter() # Start timer before potential reset/execute
1909 | reset_ack_data: Optional[Dict] = None # To store the ack from the reset call
1910 |
1911 | # --- Handle Reset Request ---
1912 | if reset:
1913 | logger.info(f"Resetting REPL state for session: {session_id}")
1914 | try:
1915 | # Assuming PyodideSandbox has a method like this that uses the direct callback
1916 | reset_ack_data = await sb.reset_repl_state() # This should wait for the JS ack
1917 | if not reset_ack_data or not reset_ack_data.get("ok"):
1918 | # Log warning but don't necessarily fail the whole call yet
1919 | error_msg = (
1920 | _format_sandbox_error(reset_ack_data.get("error"))
1921 | if reset_ack_data
1922 | else "No confirmation received"
1923 | )
1924 | logger.warning(
1925 | f"REPL state reset failed or unconfirmed for session {session_id}: {error_msg}"
1926 | )
1927 | # Optionally add this warning to the final result?
1928 | except Exception as e:
1929 | # Handle errors during the reset call itself
1930 | logger.warning(
1931 | f"Error during REPL reset call for session {session_id}: {e}", exc_info=True
1932 | )
1933 | # Store this error to potentially include in final result if no code is run
1934 | reset_ack_data = {"ok": False, "error": {"type": "ResetHostError", "message": str(e)}}
1935 |
1936 | # If ONLY resetting (no code provided), return immediately after reset attempt
1937 | if not code:
1938 | host_wall_ms = int((time.perf_counter() - t0) * 1000)
1939 | final_result = {
1940 | "success": reset_ack_data.get("ok", False)
1941 | if reset_ack_data
1942 | else False, # Reflect reset success
1943 | "stdout": "",
1944 | "stderr": "",
1945 | "result": None,
1946 | "elapsed_py_ms": 0,
1947 | "elapsed_wall_ms": host_wall_ms, # Only host time available
1948 | "handle": session_id,
1949 | "error_message": None
1950 | if (reset_ack_data and reset_ack_data.get("ok"))
1951 | else _format_sandbox_error(reset_ack_data.get("error") if reset_ack_data else None),
1952 | "error_details": reset_ack_data.get("error")
1953 | if (reset_ack_data and not reset_ack_data.get("ok"))
1954 | else None,
1955 | "reset_status": reset_ack_data, # Always include reset ack if reset was true
1956 | }
1957 | return final_result
1958 |
1959 | # --- Execute Code (if provided) ---
1960 | data: Dict[str, Any] = {} # Initialize data dict for execution results
1961 | execution_successful_this_call = True # Assume success unless execution fails
1962 | if code:
1963 | try:
1964 | # Call the execute method, ensuring repl_mode=True is passed
1965 | data = await sb.execute(
1966 | code, packages_normalized, wheels_normalized, timeout_ms, repl_mode=True
1967 | )
1968 | execution_successful_this_call = data.get(
1969 | "ok", False
1970 | ) # Get success status from execution result
1971 | except Exception as e:
1972 | # Catch host-side errors during the execute call
1973 | execution_successful_this_call = False
1974 | wall_ms_host_error = int((time.perf_counter() - t0) * 1000)
1975 | logger.error(
1976 | f"Unexpected host error calling REPL sandbox execute for {session_id}: {e}",
1977 | exc_info=True,
1978 | )
1979 | # Create a failure structure similar to what execute returns
1980 | data = {
1981 | "ok": False,
1982 | "error": {
1983 | "type": "HostExecutionError",
1984 | "message": f"Host error during REPL exec: {e}",
1985 | },
1986 | "wall_ms": wall_ms_host_error, # Use host time
1987 | "elapsed": 0,
1988 | "stdout": "",
1989 | "stderr": "",
1990 | "result": None,
1991 | }
1992 |
1993 | # --- Format results and potential errors ---
1994 | wall_ms_host_final = int((time.perf_counter() - t0) * 1000)
1995 | js_wall_ms = int(data.get("wall_ms", 0)) # Wall time reported by JS sandbox handler
1996 | py_elapsed_ms = int(data.get("elapsed", 0))
1997 | stdout_content = data.get("stdout", "")
1998 | stderr_content = data.get("stderr", "")
1999 | result_val = data.get("result")
2000 | error_info = data.get("error") # Original error dict from sandbox execution
2001 | error_message_for_caller = None
2002 | error_code_for_caller = "UnknownError"
2003 |
2004 | if not execution_successful_this_call:
2005 | error_message_for_caller = _format_sandbox_error(error_info)
2006 | if isinstance(error_info, dict):
2007 | error_code_for_caller = error_info.get("type", "UnknownSandboxError") # noqa: F841
2008 |
2009 | # --- Logging ---
2010 | action_desc = "executed" if code else "accessed (no code run)"
2011 | action_desc += " with reset" if reset else ""
2012 | log_details = {
2013 | "session_id": session_id,
2014 | "action": action_desc,
2015 | "reset_requested": reset,
2016 | "reset_successful": reset_ack_data.get("ok") if reset_ack_data else None,
2017 | "elapsed_wall_ms_host": wall_ms_host_final,
2018 | "elapsed_wall_ms_js": js_wall_ms,
2019 | "elapsed_py_ms": py_elapsed_ms,
2020 | "packages_requested": packages or [],
2021 | "packages_loaded": packages_normalized,
2022 | "wheels_count": len(wheels_normalized),
2023 | "stdout_len": len(stdout_content),
2024 | "stderr_len": len(stderr_content),
2025 | "result_type": type(result_val).__name__,
2026 | "success_this_call": execution_successful_this_call,
2027 | "repl_mode": True,
2028 | }
2029 | log_level = logger.success if execution_successful_this_call else logger.warning
2030 | log_level(
2031 | f"Python code {action_desc} in REPL sandbox (session: {session_id})",
2032 | TaskType.CODE_EXECUTION, # Assumes TaskType is defined/imported
2033 | **log_details,
2034 | error_details=error_info if not execution_successful_this_call else None,
2035 | )
2036 |
2037 | # --- Construct final return dictionary ---
2038 | final_result = {
2039 | "success": execution_successful_this_call, # Reflect success of *this* call
2040 | "stdout": stdout_content,
2041 | "stderr": stderr_content,
2042 | "result": result_val,
2043 | "elapsed_py_ms": py_elapsed_ms,
2044 | "elapsed_wall_ms": js_wall_ms or wall_ms_host_final, # Prefer JS wall time
2045 | "handle": session_id, # Always return the handle
2046 | "error_message": error_message_for_caller, # Formatted string or None
2047 | "error_details": error_info
2048 | if not execution_successful_this_call
2049 | else None, # Original dict or None
2050 | }
2051 | # Include reset status if reset was requested
2052 | if reset:
2053 | final_result["reset_status"] = reset_ack_data
2054 |
2055 | # Do NOT raise ToolError for standard Python errors caught inside sandbox,
2056 | # return them in the dictionary structure instead.
2057 | # Only raise ToolError for host-level/unrecoverable issues earlier.
2058 | return final_result
2059 |
2060 |
2061 | ###############################################################################
2062 | # Optional: Asset Preloader Function (Integrated)
2063 | ###############################################################################
2064 |
2065 |
2066 | def _get_pyodide_asset_list_from_manifest(manifest_url: str) -> List[str]:
2067 | """
2068 | Generates a list of essential Pyodide assets to preload based on version.
2069 | For v0.27.5, uses a hardcoded list as repodata.json isn't typically used
2070 | for core file listing in the same way older versions might have.
2071 | """
2072 | global _PYODIDE_VERSION # Access the global version
2073 | logger.info(f"[Preload] Generating asset list for Pyodide v{_PYODIDE_VERSION}.")
2074 |
2075 | # Version-specific logic (can be expanded for other versions)
2076 | if _PYODIDE_VERSION.startswith("0.27."):
2077 | # Hardcoded list for v0.27.x - VERIFY these against the actual CDN structure for 0.27.5!
2078 | # These are the most common core files needed for initialization.
2079 | core_files = {
2080 | # --- Core Runtime ---
2081 | "pyodide.js", # Main JS loader (UMD potentially)
2082 | "pyodide.mjs", # Main JS loader (ESM)
2083 | "pyodide.asm.js", # Wasm loader fallback/glue
2084 | "pyodide.asm.wasm", # The main WebAssembly module
2085 | # --- Standard Library ---
2086 | "python_stdlib.zip", # Packed standard library
2087 | # --- Metadata/Lock Files ---
2088 | "pyodide-lock.json", # Package lock file (crucial for loadPackage)
2089 | # --- Potential Depencencies (Less common to preload, but check CDN) ---
2090 | # "distutils.tar",
2091 | # "pyodide_py.tar",
2092 | }
2093 | logger.info(f"[Preload] Using hardcoded core asset list for v{_PYODIDE_VERSION}.")
2094 | # Log the URL that was passed but ignored for clarity
2095 | if (
2096 | manifest_url != f"{_CDN_BASE}/repodata.json"
2097 | ): # Only log if it differs from default assumption
2098 | logger.warning(f"[Preload] Ignoring provided manifest_url: {manifest_url}")
2099 | else:
2100 | # Placeholder for potentially different logic for other versions
2101 | # (e.g., actually fetching and parsing repodata.json if needed)
2102 | logger.warning(
2103 | f"[Preload] No specific asset list logic for Pyodide v{_PYODIDE_VERSION}. Using empty list."
2104 | )
2105 | core_files = set()
2106 | # If you needed to fetch/parse repodata.json for other versions:
2107 | # try:
2108 | # logger.info(f"[Preload] Fetching manifest from {manifest_url}")
2109 | # # ... (fetch manifest_url using _fetch_asset_sync or urllib) ...
2110 | # # ... (parse JSON) ...
2111 | # # ... (extract file names based on manifest structure) ...
2112 | # except Exception as e:
2113 | # logger.error(f"[Preload] Failed to fetch or parse manifest {manifest_url}: {e}")
2114 | # core_files = set() # Fallback to empty on error
2115 |
2116 | if not core_files:
2117 | logger.warning("[Preload] The generated core asset list is empty!")
2118 | else:
2119 | logger.debug(f"[Preload] Identified {len(core_files)} essential core files to fetch.")
2120 |
2121 | # Common packages are loaded on demand *within* the sandbox, not typically preloaded here.
2122 | # Explicitly state this.
2123 | logger.info(
2124 | "[Preload] Common packages (like numpy, pandas) are NOT included in this core preload list. "
2125 | "They will be fetched on demand by the sandbox if needed and cached separately "
2126 | "when `loadPackage` is called."
2127 | )
2128 |
2129 | # Return the sorted list of unique filenames
2130 | return sorted(list(core_files))
2131 |
2132 |
2133 | def preload_pyodide_assets(force_download: bool = False):
2134 | """Downloads Pyodide assets to the local cache directory."""
2135 | print("-" * 60)
2136 | print("Starting Pyodide Asset Preloader")
2137 | print(f"Target Pyodide Version: {_PYODIDE_VERSION}")
2138 | print(f"CDN Base URL: {_CDN_BASE}")
2139 | print(f"Cache Directory: {_CACHE_DIR}")
2140 | print(f"Force Re-download: {force_download}")
2141 | print("-" * 60)
2142 | try:
2143 | _CACHE_DIR.mkdir(parents=True, exist_ok=True)
2144 | except OSError as e:
2145 | print(
2146 | f"ERROR: Failed to ensure cache directory exists at {_CACHE_DIR}: {e}\nPreloading cannot proceed."
2147 | )
2148 | return
2149 | manifest_url = f"{_CDN_BASE}/repodata.json" # Dummy URL for v0.27.5 preloader logic
2150 | asset_files = _get_pyodide_asset_list_from_manifest(manifest_url)
2151 | if not asset_files:
2152 | print("ERROR: No asset files were identified.\nPreloading cannot proceed.")
2153 | return
2154 | print(f"\nAttempting to cache/verify {len(asset_files)} assets...")
2155 | cached_count = 0
2156 | verified_count = 0
2157 | error_count = 0
2158 | total_bytes_downloaded = 0
2159 | total_bytes_verified = 0
2160 | max_age = 0 if force_download else (10 * 365 * 24 * 3600)
2161 | num_files = len(asset_files)
2162 | width = len(str(num_files))
2163 | for i, filename in enumerate(asset_files):
2164 | if not filename:
2165 | logger.warning(f"[Preload] Skipping empty filename at index {i}.")
2166 | continue
2167 | file_url = f"{_CDN_BASE}/{filename}"
2168 | progress = f"[{i + 1:>{width}}/{num_files}]"
2169 | local_file_path = _local_path(file_url)
2170 | file_exists = local_file_path.exists()
2171 | is_stale = False
2172 | action = "Fetching"
2173 | if file_exists:
2174 | try:
2175 | file_stat = local_file_path.stat()
2176 | if file_stat.st_size == 0:
2177 | logger.warning(
2178 | f"[Preload] Cached file {local_file_path} is empty. Will re-fetch."
2179 | )
2180 | file_exists = False
2181 | else:
2182 | file_age = time.time() - file_stat.st_mtime
2183 | if file_age >= max_age:
2184 | is_stale = True
2185 | else:
2186 | action = "Verifying" if not force_download else "Re-fetching (forced)"
2187 | except OSError as stat_err:
2188 | logger.warning(
2189 | f"[Preload] Error checking status of {local_file_path}: {stat_err}. Will re-fetch."
2190 | )
2191 | file_exists = False
2192 | action = "Fetching (stat failed)"
2193 | if file_exists and is_stale and not force_download:
2194 | action = "Re-fetching (stale)"
2195 | display_name = filename if len(filename) <= 60 else filename[:57] + "..."
2196 | print(f"{progress} {action:<25} {display_name:<60} ... ", end="", flush=True)
2197 | try:
2198 | data = _fetch_asset_sync(file_url, max_age_s=max_age)
2199 | file_size_kb = len(data) // 1024
2200 | if action == "Verifying":
2201 | verified_count += 1
2202 | total_bytes_verified += len(data)
2203 | print(f"OK (cached, {file_size_kb:>5} KB)")
2204 | else:
2205 | cached_count += 1
2206 | total_bytes_downloaded += len(data)
2207 | status = "OK" if action.startswith("Fetch") else "OK (updated)"
2208 | print(f"{status} ({file_size_kb:>5} KB)")
2209 | except Exception as e:
2210 | print(f"ERROR: {e}")
2211 | logger.error(f"[Preload] Failed to fetch/cache {file_url}: {e}", exc_info=False)
2212 | error_count += 1
2213 | print("\n" + "-" * 60)
2214 | print("Preload Summary")
2215 | print("-" * 60)
2216 | print(f"Assets already cached & verified: {verified_count}")
2217 | print(f"Assets newly downloaded/updated: {cached_count}")
2218 | print(f"Total assets processed: {verified_count + cached_count}")
2219 | print(f"Errors encountered: {error_count}")
2220 | print("-" * 60)
2221 | print(f"Size of verified assets: {total_bytes_verified / (1024 * 1024):,.1f} MB")
2222 | print(f"Size of downloaded assets: {total_bytes_downloaded / (1024 * 1024):,.1f} MB")
2223 | print(
2224 | f"Total cache size (approx): {(total_bytes_verified + total_bytes_downloaded) / (1024 * 1024):,.1f} MB"
2225 | )
2226 | print("-" * 60)
2227 | if error_count == 0:
2228 | print("Preloading completed successfully. Assets should be cached for offline use.")
2229 | else:
2230 | print(
2231 | f"WARNING: {error_count} assets failed to download. Offline functionality may be incomplete."
2232 | )
2233 | print("-" * 60)
2234 |
2235 |
2236 | ###############################################################################
2237 | # Main execution block for preloading (if script is run directly)
2238 | ###############################################################################
2239 | if __name__ == "__main__":
2240 | # Setup logging if run as main script
2241 | logging.basicConfig(
2242 | level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
2243 | )
2244 |
2245 | parser = argparse.ArgumentParser(
2246 | description="Utility for the Python Sandbox module. Includes Pyodide asset preloader.",
2247 | formatter_class=argparse.RawDescriptionHelpFormatter,
2248 | epilog=""" Examples:\n Cache Pyodide assets (download if missing/stale):\n python %(prog)s --preload\n\n Force re-download of all assets, ignoring cache:\n python %(prog)s --preload --force """,
2249 | )
2250 | parser.add_argument(
2251 | "--preload",
2252 | action="store_true",
2253 | help="Run the Pyodide asset preloader to cache files required for offline operation.",
2254 | )
2255 | parser.add_argument(
2256 | "--force",
2257 | "-f",
2258 | action="store_true",
2259 | help="Force re-download of all assets during preload, ignoring existing cache validity.",
2260 | )
2261 | args = parser.parse_args()
2262 | if args.preload:
2263 | preload_pyodide_assets(force_download=args.force)
2264 | else:
2265 | print(
2266 | "This script contains the PythonSandbox tool implementation.\nUse the --preload argument to cache Pyodide assets for offline use.\nExample: python path/to/python_sandbox.py --preload"
2267 | )
2268 |
```