This is page 3 of 4. Use http://codebase.md/ilikepizza2/qa-mcp?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .env.example
├── .gitignore
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE
├── main.py
├── mcp_server.py
├── README.md
├── requirements.txt
├── src
│ ├── __init__.py
│ ├── agents
│ │ ├── __init__.py
│ │ ├── auth_agent.py
│ │ ├── crawler_agent.py
│ │ ├── js_utils
│ │ │ └── xpathgenerator.js
│ │ └── recorder_agent.py
│ ├── browser
│ │ ├── __init__.py
│ │ ├── browser_controller.py
│ │ └── panel
│ │ └── panel.py
│ ├── core
│ │ ├── __init__.py
│ │ └── task_manager.py
│ ├── dom
│ │ ├── buildDomTree.js
│ │ ├── history
│ │ │ ├── service.py
│ │ │ └── view.py
│ │ ├── service.py
│ │ └── views.py
│ ├── execution
│ │ ├── __init__.py
│ │ └── executor.py
│ ├── llm
│ │ ├── __init__.py
│ │ ├── clients
│ │ │ ├── azure_openai_client.py
│ │ │ ├── gemini_client.py
│ │ │ └── openai_client.py
│ │ └── llm_client.py
│ ├── security
│ │ ├── __init__.py
│ │ ├── nuclei_scanner.py
│ │ ├── semgrep_scanner.py
│ │ ├── utils.py
│ │ └── zap_scanner.py
│ └── utils
│ ├── __init__.py
│ ├── image_utils.py
│ └── utils.py
└── test_schema.md
```
# Files
--------------------------------------------------------------------------------
/src/browser/browser_controller.py:
--------------------------------------------------------------------------------
```python
1 | # /src/browser/browser_controller.py
2 | from patchright.sync_api import sync_playwright, Page, Browser, Playwright, TimeoutError as PlaywrightTimeoutError, Error as PlaywrightError, Response, Request, Locator, ConsoleMessage, expect
3 | import logging
4 | import time
5 | import random
6 | import json
7 | import os
8 | from typing import Optional, Any, Dict, List, Callable, Tuple
9 | import threading
10 | import platform
11 |
12 | from ..dom.service import DomService
13 | from ..dom.views import DOMState, DOMElementNode, SelectorMap
14 | from .panel.panel import Panel
15 |
16 | logger = logging.getLogger(__name__)
17 |
18 | COMMON_HEADERS = {
19 | 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
20 | 'Accept-Encoding': 'gzip, deflate, br',
21 | 'Accept-Language': 'en-US,en;q=0.9',
22 | 'Sec-Fetch-Dest': 'document',
23 | 'Sec-Fetch-Mode': 'navigate',
24 | 'Sec-Fetch-Site': 'none',
25 | 'Sec-Fetch-User': '?1',
26 | 'Upgrade-Insecure-Requests': '1',
27 | }
28 |
29 | HIDE_WEBDRIVER_SCRIPT = """
30 | Object.defineProperty(navigator, 'webdriver', {
31 | get: () => undefined
32 | });
33 | """
34 |
35 | # --- JavaScript for click listener and selector generation ---
36 | CLICK_LISTENER_JS = """
37 | async () => {
38 | // Reset/Initialize the global flag/variable
39 | window._recorder_override_selector = undefined;
40 | console.log('[Recorder Listener] Attaching click listener...');
41 | const PANEL_ID = 'bw-recorder-panel';
42 |
43 | const clickHandler = async (event) => {
44 | const targetElement = event.target;
45 | let isPanelClick = false; // Flag to track if click is inside the panel
46 |
47 | // Check if the click target is the panel or inside the panel
48 | // using element.closest()
49 | if (targetElement && targetElement.closest && targetElement.closest(`#${PANEL_ID}`)) {
50 | console.log('[Recorder Listener] Click inside panel detected. Allowing event to proceed normally.');
51 | isPanelClick = true;
52 | // DO ABSOLUTELY NOTHING HERE - let the event continue to the button's own listener
53 | }
54 |
55 | // --- Only process as an override attempt if it was NOT a panel click ---
56 | if (!isPanelClick) {
57 | console.log('[Recorder Listener] Click detected (Outside panel)! Processing as override.');
58 | event.preventDefault(); // Prevent default action (like navigation) ONLY for override clicks
59 | event.stopPropagation(); // Stop propagation ONLY for override clicks
60 |
61 | if (!targetElement) {
62 | console.warn('[Recorder Listener] Override click event has no target.');
63 | // Remove listener even if target is null to avoid getting stuck
64 | document.body.removeEventListener('click', clickHandler, { capture: true });
65 | console.log('[Recorder Listener] Listener removed due to null target.');
66 | return;
67 | }
68 |
69 |
70 | // --- Simple Selector Generation (enhance as needed) ---
71 | let selector = '';
72 | function escapeCSS(value) {
73 | if (!value) return '';
74 | // Basic escape for common CSS special chars in identifiers/strings
75 | // For robust escaping, a library might be better, but this covers many cases.
76 | return value.replace(/([!"#$%&'()*+,./:;<=>?@\\[\\]^`{|}~])/g, '\\$1');
77 | }
78 |
79 | if (targetElement.id && targetElement.id !== PANEL_ID && targetElement.id !== 'playwright-highlight-container') {
80 | selector = `#${escapeCSS(targetElement.id.trim())}`;
81 | } else if (targetElement.getAttribute('data-testid')) {
82 | selector = `[data-testid="${escapeCSS(targetElement.getAttribute('data-testid').trim())}"]`;
83 | } else if (targetElement.name) {
84 | selector = `${targetElement.tagName.toLowerCase()}[name="${escapeCSS(targetElement.name.trim())}"]`;
85 | } else {
86 | // Fallback: Basic XPath -> CSS approximation (needs improvement)
87 | let path = '';
88 | let current = targetElement;
89 | while (current && current.tagName && current.tagName.toLowerCase() !== 'body' && current.parentNode) {
90 | let segment = current.tagName.toLowerCase();
91 | const parent = current.parentElement;
92 | if (parent) {
93 | const siblings = Array.from(parent.children);
94 | const sameTagSiblings = siblings.filter(sib => sib.tagName === current.tagName);
95 | if (sameTagSiblings.length > 1) {
96 | let index = 1;
97 | for(let i=0; i < sameTagSiblings.length; i++) { // Find index correctly
98 | if(sameTagSiblings[i] === current) {
99 | index = i + 1;
100 | break;
101 | }
102 | }
103 | // Prefer nth-child if possible, might be slightly more stable
104 | try {
105 | const siblingIndex = Array.prototype.indexOf.call(parent.children, current) + 1;
106 | segment += `:nth-child(${siblingIndex})`;
107 | } catch(e) { // Fallback if indexOf fails
108 | segment += `:nth-of-type(${index})`;
109 | }
110 | }
111 | }
112 | path = segment + (path ? ' > ' + path : '');
113 | current = parent;
114 | }
115 | selector = path ? `body > ${path}` : targetElement.tagName.toLowerCase();
116 | console.log(`[Recorder Listener] Generated fallback selector: ${selector}`);
117 | }
118 | // --- End Selector Generation ---
119 |
120 | console.log(`[Recorder Listener] Override Target: ${targetElement.tagName}, Generated selector: ${selector}`);
121 |
122 | // Only set override if a non-empty selector was generated
123 | if (selector) {
124 | window._recorder_override_selector = selector;
125 | console.log('[Recorder Listener] Override selector variable set.');
126 | } else {
127 | console.warn('[Recorder Listener] Could not generate a valid selector for the override click.');
128 | }
129 |
130 | // ---- IMPORTANT: Remove the listener AFTER processing an override click ----
131 | // This prevents it interfering further and ensures it's gone before panel interaction waits
132 | document.body.removeEventListener('click', clickHandler, { capture: true });
133 | console.log('[Recorder Listener] Listener removed after processing override click.');
134 | };
135 | // If it WAS a panel click (isPanelClick = true), we did nothing in this handler.
136 | // The event continues to the button's specific onclick handler.
137 | // The listener remains attached to the body for subsequent clicks outside the panel.
138 | };
139 | // --- Add listener ---
140 | // Ensure no previous listener exists before adding a new one
141 | if (window._recorderClickListener) {
142 | console.warn('[Recorder Listener] Removing potentially lingering listener before attaching new one.');
143 | document.body.removeEventListener('click', window._recorderClickListener, { capture: true });
144 | }
145 | // Add listener in capture phase to catch clicks first
146 | document.body.addEventListener('click', clickHandler, { capture: true });
147 | window._recorderClickListener = clickHandler; // Store reference to remove later
148 | }
149 | """
150 |
151 | REMOVE_CLICK_LISTENER_JS = """
152 | () => {
153 | let removed = false;
154 | // Remove listener
155 | if (window._recorderClickListener) {
156 | document.body.removeEventListener('click', window._recorderClickListener, { capture: true });
157 | delete window._recorderClickListener;
158 | console.log('[Recorder Listener] Listener explicitly removed.');
159 | removed = true;
160 | } else {
161 | console.log('[Recorder Listener] No active listener found to remove.');
162 | }
163 | // Clean up global variable
164 | if (window._recorder_override_selector !== undefined) {
165 | delete window._recorder_override_selector;
166 | console.log('[Recorder Listener] Override selector variable cleaned up.');
167 | }
168 | return removed;
169 | }
170 | """
171 |
172 |
173 | class BrowserController:
174 | """Handles Playwright browser automation tasks, including console message capture."""
175 |
176 | def __init__(self, headless=True, viewport_size=None, auth_state_path: Optional[str] = None):
177 | self.playwright: Playwright | None = None
178 | self.browser: Browser | None = None
179 | self.context: Optional[Any] = None # Keep context reference
180 | self.page: Page | None = None
181 | self.headless = headless
182 | self.default_navigation_timeout = 9000
183 | self.default_action_timeout = 9000
184 | self._dom_service: Optional[DomService] = None
185 | self.console_messages: List[Dict[str, Any]] = [] # <-- Add list to store messages
186 | self.viewport_size = viewport_size
187 | self.network_requests: List[Dict[str, Any]] = []
188 | self.page_performance_timing: Optional[Dict[str, Any]] = None
189 | self.auth_state_path = auth_state_path
190 |
191 | self.panel = Panel(headless=headless, page=self.page)
192 | logger.info(f"BrowserController initialized (headless={headless}).")
193 |
194 | def _handle_response(self, response: Response):
195 | """Callback function to handle network responses."""
196 | request = response.request
197 | timing = request.timing
198 | # Calculate duration robustly
199 | start_time = timing.get('requestStart', -1)
200 | end_time = timing.get('responseEnd', -1)
201 | duration_ms = None
202 | if start_time >= 0 and end_time >= 0 and end_time >= start_time:
203 | duration_ms = round(end_time - start_time)
204 |
205 | req_data = {
206 | "url": response.url,
207 | "method": request.method,
208 | "status": response.status,
209 | "status_text": response.status_text,
210 | "start_time_ms": start_time if start_time >= 0 else None, # Use ms relative to navigationStart
211 | "end_time_ms": end_time if end_time >= 0 else None, # Use ms relative to navigationStart
212 | "duration_ms": duration_ms,
213 | "resource_type": request.resource_type,
214 | "headers": dict(response.headers), # Store response headers
215 | "request_headers": dict(request.headers), # Store request headers
216 | # Timing breakdown (optional, can be verbose)
217 | # "timing_details": timing,
218 | }
219 | self.network_requests.append(req_data)
220 |
221 | def _handle_request_failed(self, request: Request):
222 | """Callback function to handle failed network requests."""
223 | try:
224 | failure_text = request.failure
225 | logger.warning(f"[NETWORK.FAILED] {request.method} {request.url} - Error: {failure_text}")
226 | req_data = {
227 | "url": request.url,
228 | "method": request.method,
229 | "status": None, # No status code available for request failure typically
230 | "status_text": "Request Failed",
231 | "start_time_ms": request.timing.get('requestStart', -1) if request.timing else None, # May still have start time
232 | "end_time_ms": None, # Failed before response end
233 | "duration_ms": None,
234 | "resource_type": request.resource_type,
235 | "headers": None, # No response headers
236 | "request_headers": dict(request.headers),
237 | "error_text": failure_text # Store the failure reason
238 | }
239 | self.network_requests.append(req_data)
240 | except Exception as e:
241 | logger.error(f"Error within _handle_request_failed for URL {request.url}: {e}", exc_info=True)
242 |
243 | def _handle_console_message(self, message: ConsoleMessage):
244 | """Callback function to handle console messages."""
245 | msg_type = message.type
246 | msg_text = message.text
247 | timestamp = time.time()
248 | log_entry = {
249 | "timestamp": timestamp,
250 | "type": msg_type,
251 | "text": msg_text,
252 | # Optional: Add location if needed, but can be verbose
253 | # "location": message.location()
254 | }
255 | self.console_messages.append(log_entry)
256 | # Optional: Log immediately to agent's log file for real-time debugging
257 | log_level = logging.WARNING if msg_type in ['error', 'warning'] else logging.DEBUG
258 | logger.log(log_level, f"[CONSOLE.{msg_type.upper()}] {msg_text}")
259 |
260 | def _get_random_user_agent(self):
261 | """Provides a random choice from a list of common user agents."""
262 | user_agents = [
263 | # Chrome on Windows
264 | 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
265 | 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
266 | 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
267 | # Chrome on Mac
268 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
269 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
270 | # Firefox on Windows
271 | 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/119.0',
272 | # Add more variations if desired (Edge, Safari etc.)
273 | ]
274 | return random.choice(user_agents)
275 |
276 | def _get_random_viewport(self):
277 | """Provides a slightly randomized common viewport size."""
278 | common_sizes = [
279 | # {'width': 1280, 'height': 720},
280 | # {'width': 1366, 'height': 768},
281 | {'width': 800, 'height': 600},
282 | # {'width': 1536, 'height': 864},
283 | ]
284 | base = random.choice(common_sizes)
285 | # Add small random offset
286 | if not self.viewport_size:
287 | base['width'] += random.randint(-10, 10)
288 | base['height'] += random.randint(-5, 5)
289 | else:
290 | base = self.viewport_size
291 | return base
292 |
293 | def _human_like_delay(self, min_secs: float, max_secs: float):
294 | """ Sleeps for a random duration within the specified range. """
295 | delay = random.uniform(min_secs, max_secs)
296 | logger.debug(f"Applying human-like delay: {delay:.2f} seconds")
297 | time.sleep(delay)
298 |
299 | def _get_locator(self, selector: str):
300 | """
301 | Gets a Playwright locator for the first matching element,
302 | handling potential XPath selectors passed as CSS.
303 | """
304 | if not self.page:
305 | raise PlaywrightError("Page is not initialized.")
306 | if not selector:
307 | raise ValueError("Selector cannot be empty.")
308 |
309 | # Basic check to see if it looks like XPath
310 | # Playwright's locator handles 'xpath=...' automatically,
311 | # but sometimes plain XPaths are passed. Let's try to detect them.
312 | is_likely_xpath = selector.startswith(('/', '(', '.')) or \
313 | ('/' in selector and not any(c in selector for c in ['#', '.', '[', '>', '+', '~', '='])) # Avoid CSS chars
314 |
315 | processed_selector = selector
316 | if is_likely_xpath and not selector.startswith(('css=', 'xpath=')):
317 | # If it looks like XPath, explicitly prefix it for Playwright's locator
318 | logger.debug(f"Selector '{selector}' looks like XPath. Using explicit 'xpath=' prefix.")
319 | processed_selector = f"xpath={selector}"
320 | # If it starts with css= or xpath=, Playwright handles it.
321 | # Otherwise, it's assumed to be a CSS selector.
322 |
323 | try:
324 | logger.debug(f"Attempting to create locator using: '{processed_selector}'")
325 | # Use .first to always target a single element, consistent with other actions
326 | locator = self.page.locator(processed_selector).first
327 | return locator
328 | except Exception as e:
329 | # Catch errors during locator creation itself (e.g., invalid selector syntax)
330 | logger.error(f"Failed to create locator for processed selector: '{processed_selector}'. Original: '{selector}'. Error: {e}")
331 | # Re-raise using the processed selector in the message for clarity
332 | raise PlaywrightError(f"Invalid selector syntax or error creating locator: '{processed_selector}'. Error: {e}") from e
333 |
334 |
335 | # Recorder Methods =============
336 | def setup_click_listener(self) -> bool:
337 | """Injects JS to listen for the next user click and report the selector."""
338 | if self.headless:
339 | logger.error("Cannot set up click listener in headless mode.")
340 | return False
341 | if not self.page:
342 | logger.error("Page not initialized. Cannot set up click listener.")
343 | return False
344 | try:
345 | # Inject and run the listener setup JS
346 | # It now resets the flag internally before adding the listener
347 | self.page.evaluate(CLICK_LISTENER_JS)
348 | logger.info("JavaScript click listener attached (using pre-exposed callback).")
349 | return True
350 |
351 | except Exception as e:
352 | logger.error(f"Failed to set up recorder click listener: {e}", exc_info=True)
353 | return False
354 |
355 | def remove_click_listener(self) -> bool:
356 | """Removes the injected JS click listener."""
357 | if self.headless: return True # Nothing to remove
358 | if not self.page:
359 | logger.warning("Page not initialized. Cannot remove click listener.")
360 | return False
361 | try:
362 | removed = self.page.evaluate(REMOVE_CLICK_LISTENER_JS)
363 |
364 | return removed
365 | except Exception as e:
366 | logger.error(f"Failed to remove recorder click listener: {e}", exc_info=True)
367 | return False
368 |
369 | def wait_for_user_click_or_timeout(self, timeout_seconds: float) -> Optional[str]:
370 | """
371 | Waits for the user to click (triggering the callback) or for the timeout.
372 | Returns the selector if clicked, None otherwise.
373 | MUST be called after setup_click_listener.
374 | """
375 | if self.headless: return None
376 | if not self.page:
377 | logger.error("Page not initialized. Cannot wait for click function.")
378 | return None
379 |
380 | selector_result = None
381 | js_condition = "() => window._recorder_override_selector !== undefined"
382 | timeout_ms = timeout_seconds * 1000
383 |
384 | logger.info(f"Waiting up to {timeout_seconds}s for user click (checking JS flag)...")
385 |
386 | try:
387 | # Wait for the JS condition to become true
388 | self.page.wait_for_function(js_condition, timeout=timeout_ms)
389 |
390 | # If wait_for_function completes without timeout, the flag was set
391 | logger.info("User click detected (JS flag set)!")
392 | # Retrieve the value set by the click handler
393 | selector_result = self.page.evaluate("window._recorder_override_selector")
394 | logger.debug(f"Retrieved selector from JS flag: {selector_result}")
395 |
396 | except PlaywrightTimeoutError:
397 | logger.info("Timeout reached waiting for user click (JS flag not set).")
398 | selector_result = None # Timeout occurred
399 | except Exception as e:
400 | logger.error(f"Error during page.wait_for_function: {e}", exc_info=True)
401 | selector_result = None # Treat other errors as timeout/failure
402 |
403 | finally:
404 | # Clean up the JS listener and the flag regardless of outcome
405 | self.remove_click_listener()
406 |
407 | return selector_result
408 |
409 |
410 | # Highlighting elements
411 | def highlight_element(self, selector: str, index: int, color: str = "#FF0000", text: Optional[str] = None, node_xpath: Optional[str] = None):
412 | """Highlights an element using a specific selector and index label."""
413 | if self.headless or not self.page: return
414 | try:
415 | self.page.evaluate("""
416 | (args) => {
417 | const { selector, index, color, text, node_xpath } = args;
418 | const HIGHLIGHT_CONTAINER_ID = "bw-highlight-container"; // Unique ID
419 |
420 | let container = document.getElementById(HIGHLIGHT_CONTAINER_ID);
421 | if (!container) {
422 | container = document.createElement("div");
423 | container.id = HIGHLIGHT_CONTAINER_ID;
424 | container.style.position = "fixed";
425 | container.style.pointerEvents = "none";
426 | container.style.top = "0";
427 | container.style.left = "0";
428 | container.style.width = "0"; // Occupy no space
429 | container.style.height = "0";
430 | container.style.zIndex = "2147483646"; // Below listener potentially
431 | document.body.appendChild(container);
432 | }
433 |
434 | let element = null;
435 | try {
436 | element = document.querySelector(selector);
437 | } catch (e) {
438 | console.warn(`[Highlighter] querySelector failed for '${selector}': ${e.message}.`);
439 | element = null; // Ensure element is null if querySelector fails
440 | }
441 |
442 | // --- Fallback to XPath if CSS failed AND xpath is available ---
443 | if (!element && node_xpath) {
444 | console.log(`[Highlighter] Falling back to XPath: ${node_xpath}`);
445 | try {
446 | element = document.evaluate(
447 | node_xpath,
448 | document,
449 | null,
450 | XPathResult.FIRST_ORDERED_NODE_TYPE,
451 | null
452 | ).singleNodeValue;
453 | } catch (e) {
454 | console.error(`[Highlighter] XPath evaluation failed for '${node_xpath}': ${e.message}`);
455 | element = null;
456 | }
457 | }
458 | // ------------------------------------------------------------
459 |
460 | if (!element) {
461 | console.warn(`[Highlighter] Element not found using selector '${selector}' or XPath '${node_xpath}'. Cannot highlight.`);
462 | return;
463 | }
464 |
465 | const rect = element.getBoundingClientRect();
466 | if (!rect || rect.width === 0 || rect.height === 0) return; // Don't highlight non-rendered
467 |
468 | const overlay = document.createElement("div");
469 | overlay.style.position = "fixed";
470 | overlay.style.border = `2px solid ${color}`;
471 | overlay.style.backgroundColor = color + '1A'; // 10% opacity
472 | overlay.style.pointerEvents = "none";
473 | overlay.style.boxSizing = "border-box";
474 | overlay.style.top = `${rect.top}px`;
475 | overlay.style.left = `${rect.left}px`;
476 | overlay.style.width = `${rect.width}px`;
477 | overlay.style.height = `${rect.height}px`;
478 | overlay.style.zIndex = "2147483646";
479 | overlay.setAttribute('data-highlight-selector', selector); // Mark for cleanup
480 | container.appendChild(overlay);
481 |
482 | const label = document.createElement("div");
483 | const labelText = text ? `${index}: ${text}` : `${index}`;
484 | label.style.position = "fixed";
485 | label.style.background = color;
486 | label.style.color = "white";
487 | label.style.padding = "1px 4px";
488 | label.style.borderRadius = "4px";
489 | label.style.fontSize = "10px";
490 | label.style.fontWeight = "bold";
491 | label.style.zIndex = "2147483647";
492 | label.textContent = labelText;
493 | label.setAttribute('data-highlight-selector', selector); // Mark for cleanup
494 |
495 | // Position label top-left, slightly offset
496 | let labelTop = rect.top - 18;
497 | let labelLeft = rect.left;
498 | // Adjust if label would go off-screen top
499 | if (labelTop < 0) labelTop = rect.top + 2;
500 |
501 | label.style.top = `${labelTop}px`;
502 | label.style.left = `${labelLeft}px`;
503 | container.appendChild(label);
504 | }
505 | """, {"selector": selector, "index": index, "color": color, "text": text, "node_xpath": node_xpath})
506 | except Exception as e:
507 | logger.warning(f"Failed to highlight element '{selector}': {e}")
508 |
509 | def clear_highlights(self):
510 | """Removes all highlight overlays and labels added by highlight_element."""
511 | if self.headless or not self.page: return
512 | try:
513 | self.page.evaluate("""
514 | () => {
515 | const container = document.getElementById("bw-highlight-container");
516 | if (container) {
517 | container.innerHTML = ''; // Clear contents efficiently
518 | }
519 | }
520 | """)
521 | # logger.debug("Cleared highlights.")
522 | except Exception as e:
523 | logger.warning(f"Could not clear highlights: {e}")
524 |
525 |
526 | # Getters
527 | def get_structured_dom(self, highlight_all_clickable_elements: bool = True, viewport_expansion: int = 0) -> Optional[DOMState]:
528 | """
529 | Uses DomService to get a structured representation of the interactive DOM elements.
530 |
531 | Args:
532 | highlight_all_clickable_elements: Whether to visually highlight elements in the browser.
533 | viewport_expansion: Pixel value to expand the viewport for element detection (0=viewport only, -1=all).
534 |
535 | Returns:
536 | A DOMState object containing the element tree and selector map, or None on error.
537 | """
538 | highlight_all_clickable_elements = False # SETTING TO FALSE TO AVOID CONFUSION WITH NEXT ACTION HIGHLIGHT
539 |
540 | if not self.page:
541 | logger.error("Browser/Page not initialized or DomService unavailable.")
542 | return None
543 | if not self._dom_service:
544 | self._dom_service = DomService(self.page)
545 |
546 |
547 | # --- RECORDER MODE: Never highlight via JS during DOM build ---
548 | # Highlighting is done separately by BrowserController.highlight_element
549 | if self.headless == False: # Assume non-headless is recorder mode context
550 | highlight_all_clickable_elements = False
551 | # --- END RECORDER MODE ---
552 |
553 | if not self._dom_service:
554 | logger.error("DomService unavailable.")
555 | return None
556 |
557 | try:
558 | logger.info(f"Requesting structured DOM (highlight={highlight_all_clickable_elements}, expansion={viewport_expansion})...")
559 | start_time = time.time()
560 | dom_state = self._dom_service.get_clickable_elements(
561 | highlight_elements=highlight_all_clickable_elements,
562 | focus_element=-1, # Not focusing on a specific element for now
563 | viewport_expansion=viewport_expansion
564 | )
565 | end_time = time.time()
566 | logger.info(f"Structured DOM retrieved in {end_time - start_time:.2f}s. Found {len(dom_state.selector_map)} interactive elements.")
567 | # Generate selectors immediately for recorder use
568 | if dom_state and dom_state.selector_map:
569 | for node in dom_state.selector_map.values():
570 | if not node.css_selector:
571 | node.css_selector = self.get_selector_for_node(node)
572 | return dom_state
573 |
574 | except Exception as e:
575 | logger.error(f"Error getting structured DOM: {type(e).__name__}: {e}", exc_info=True)
576 | return None
577 |
578 | def get_selector_for_node(self, node: DOMElementNode) -> Optional[str]:
579 | """Generates a robust CSS selector for a given DOMElementNode."""
580 | if not node: return None
581 | try:
582 | # Use the static method from DomService
583 | return DomService._enhanced_css_selector_for_element(node)
584 | except Exception as e:
585 | logger.error(f"Error generating selector for node {node.xpath}: {e}", exc_info=True)
586 | return node.xpath # Fallback to xpath
587 |
588 | def get_performance_timing(self) -> Optional[Dict[str, Any]]:
589 | """Gets the window.performance.timing object from the page."""
590 | if not self.page:
591 | logger.error("Cannot get performance timing, page not initialized.")
592 | return None
593 | try:
594 | # Evaluate script to get the performance timing object as JSON
595 | timing_json = self.page.evaluate("() => JSON.stringify(window.performance.timing)")
596 | if timing_json:
597 | self.page_performance_timing = json.loads(timing_json) # Store it
598 | logger.debug("Retrieved window.performance.timing.")
599 | return self.page_performance_timing
600 | else:
601 | logger.warning("window.performance.timing unavailable or empty.")
602 | return None
603 | except Exception as e:
604 | logger.error(f"Error getting performance timing: {e}", exc_info=True)
605 | return None
606 |
607 | def get_current_url(self) -> str:
608 | """Returns the current URL of the page."""
609 | if not self.page:
610 | return "Error: Browser not started."
611 | try:
612 | return self.page.url
613 | except Exception as e:
614 | logger.error(f"Error getting current URL: {e}", exc_info=True)
615 | return f"Error retrieving URL: {e}"
616 |
617 | def get_browser_version(self) -> str:
618 | if not self.browser:
619 | return "Unknown"
620 | try:
621 | # Browser version might be available directly
622 | return f"{self.browser.browser_type.name} {self.browser.version}"
623 | except Exception:
624 | logger.warning("Could not retrieve exact browser version.")
625 | return self.browser.browser_type.name if self.browser else "Unknown"
626 |
627 | def get_os_info(self) -> str:
628 | try:
629 | return f"{platform.system()} {platform.release()}"
630 | except Exception:
631 | logger.warning("Could not retrieve OS information.")
632 | return "Unknown"
633 |
634 | def get_viewport_size(self) -> Optional[Dict[str, int]]:
635 | if not self.page:
636 | return None
637 | try:
638 | return self.page.viewport_size # Returns {'width': W, 'height': H} or None
639 | except Exception:
640 | logger.warning("Could not retrieve viewport size.")
641 | return None
642 |
643 |
644 | def get_console_messages(self) -> List[Dict[str, Any]]:
645 | """Returns a copy of the captured console messages."""
646 | return list(self.console_messages) # Return a copy
647 |
648 | def clear_console_messages(self):
649 | """Clears the stored console messages."""
650 | logger.debug("Clearing captured console messages.")
651 | self.console_messages = []
652 |
653 | def get_network_requests(self) -> List[Dict[str, Any]]:
654 | """Returns a copy of the captured network request data."""
655 | return list(self.network_requests)
656 |
657 | def clear_network_requests(self):
658 | """Clears the stored network request data."""
659 | logger.debug("Clearing captured network requests.")
660 | self.network_requests = []
661 |
662 |
663 |
664 | def validate_assertion(self, assertion_type: str, selector: str, params: Dict[str, Any], timeout_ms: int = 3000) -> Tuple[bool, Optional[str]]:
665 | """
666 | Performs a quick Playwright check to validate a proposed assertion.
667 |
668 | Args:
669 | assertion_type: The type of assertion (e.g., 'assert_visible').
670 | selector: The CSS selector for the target element.
671 | params: Dictionary of parameters for the assertion (e.g., expected_text).
672 | timeout_ms: Short timeout for the validation check.
673 |
674 | Returns:
675 | Tuple (bool, Optional[str]): (True, None) if validation passes,
676 | (False, error_message) if validation fails.
677 | """
678 | if not self.page:
679 | return False, "Page not initialized."
680 | if not selector:
681 | # Assertions like 'assert_llm_verification' might not have a selector
682 | if assertion_type == 'assert_llm_verification':
683 | logger.info("Skipping validation for 'assert_llm_verification' as it relies on external LLM check.")
684 | return True, None
685 | return False, "Selector is required for validation."
686 | if not assertion_type:
687 | return False, "Assertion type is required for validation."
688 |
689 | logger.info(f"Validating assertion: {assertion_type} on '{selector}' with params {params} (timeout: {timeout_ms}ms)")
690 | try:
691 | locator = self._get_locator(selector) # Use helper to handle xpath/css
692 |
693 | # Use Playwright's expect() for efficient checks
694 | if assertion_type == 'assert_visible':
695 | expect(locator).to_be_visible(timeout=timeout_ms)
696 | elif assertion_type == 'assert_hidden':
697 | expect(locator).to_be_hidden(timeout=timeout_ms)
698 | elif assertion_type == 'assert_text_equals':
699 | expected_text = params.get('expected_text')
700 | if expected_text is None: return False, "Missing 'expected_text' parameter for assert_text_equals"
701 | expect(locator).to_have_text(expected_text, timeout=timeout_ms)
702 | elif assertion_type == 'assert_text_contains':
703 | expected_text = params.get('expected_text')
704 | if expected_text is None: return False, "Missing 'expected_text' parameter for assert_text_contains"
705 | expect(locator).to_contain_text(expected_text, timeout=timeout_ms)
706 | elif assertion_type == 'assert_attribute_equals':
707 | attr_name = params.get('attribute_name')
708 | expected_value = params.get('expected_value')
709 | if not attr_name: return False, "Missing 'attribute_name' parameter"
710 | # Note: Playwright's to_have_attribute handles presence and value check
711 | expect(locator).to_have_attribute(attr_name, expected_value if expected_value is not None else "", timeout=timeout_ms) # Check empty string if value is None/missing? Or require value? Let's require non-None value.
712 | # if expected_value is None: return False, "Missing 'expected_value' parameter" # Stricter check
713 | # expect(locator).to_have_attribute(attr_name, expected_value, timeout=timeout_ms)
714 | elif assertion_type == 'assert_element_count':
715 | expected_count = params.get('expected_count')
716 | if expected_count is None: return False, "Missing 'expected_count' parameter"
717 | # Re-evaluate locator to get all matches for count
718 | all_matches_locator = self.page.locator(selector)
719 | expect(all_matches_locator).to_have_count(expected_count, timeout=timeout_ms)
720 | elif assertion_type == 'assert_checked':
721 | expect(locator).to_be_checked(timeout=timeout_ms)
722 | elif assertion_type == 'assert_not_checked':
723 | # Use expect(...).not_to_be_checked()
724 | expect(locator).not_to_be_checked(timeout=timeout_ms)
725 | elif assertion_type == 'assert_enabled':
726 | expect(locator).to_be_enabled(timeout=timeout_ms)
727 | elif assertion_type == 'assert_disabled':
728 | expect(locator).to_be_disabled(timeout=timeout_ms)
729 | elif assertion_type == 'assert_llm_verification':
730 | logger.info("Skipping Playwright validation for 'assert_llm_verification'.")
731 | # This assertion type is validated externally by the LLM during execution.
732 | pass # Treat as passed for this quick check
733 | else:
734 | return False, f"Unsupported assertion type for validation: {assertion_type}"
735 |
736 | # If no exception was raised by expect()
737 | logger.info(f"Validation successful for {assertion_type} on '{selector}'.")
738 | return True, None
739 |
740 | except PlaywrightTimeoutError as e:
741 | err_msg = f"Validation failed for {assertion_type} on '{selector}': Timeout ({timeout_ms}ms) - {str(e).splitlines()[0]}"
742 | logger.warning(err_msg)
743 | return False, err_msg
744 | except AssertionError as e: # Catch expect() assertion failures
745 | err_msg = f"Validation failed for {assertion_type} on '{selector}': Condition not met - {str(e).splitlines()[0]}"
746 | logger.warning(err_msg)
747 | return False, err_msg
748 | except PlaywrightError as e:
749 | err_msg = f"Validation failed for {assertion_type} on '{selector}': PlaywrightError - {str(e).splitlines()[0]}"
750 | logger.warning(err_msg)
751 | return False, err_msg
752 | except Exception as e:
753 | err_msg = f"Unexpected error during validation for {assertion_type} on '{selector}': {type(e).__name__} - {e}"
754 | logger.error(err_msg, exc_info=True)
755 | return False, err_msg
756 |
757 |
758 |
759 | def goto(self, url: str):
760 | """Navigates the page to a specific URL."""
761 | if not self.page:
762 | raise PlaywrightError("Browser not started. Call start() first.")
763 | try:
764 | logger.info(f"Navigating to URL: {url}")
765 | # Use default navigation timeout set in context
766 | response = self.page.goto(url, wait_until='load', timeout=self.default_navigation_timeout)
767 | # Add a small stable delay after load
768 | time.sleep(1)
769 | status = response.status if response else 'unknown'
770 |
771 | # --- Capture performance timing after navigation ---
772 | self.get_performance_timing()
773 |
774 | logger.info(f"Navigation to {url} finished with status: {status}.")
775 | if response and not response.ok:
776 | logger.warning(f"Navigation to {url} resulted in non-OK status: {status}")
777 | # Optionally raise an error here if needed
778 | except PlaywrightTimeoutError as e:
779 | logger.error(f"Timeout navigating to {url}: {e}")
780 | # Re-raise with a clearer message for the agent
781 | raise PlaywrightTimeoutError(f"Timeout loading page {url}. The page might be too slow or unresponsive.") from e
782 | except PlaywrightError as e: # Catch broader Playwright errors
783 | logger.error(f"Playwright error navigating to {url}: {e}")
784 | raise PlaywrightError(f"Error navigating to {url}: {e}") from e
785 | except Exception as e:
786 | logger.error(f"Unexpected error navigating to {url}: {e}", exc_info=True)
787 | raise # Re-raise for the agent to handle
788 |
789 | def check(self, selector: str):
790 | """Checks a checkbox or radio button."""
791 | if not self.page:
792 | raise PlaywrightError("Browser not started.")
793 | try:
794 | logger.info(f"Attempting to check element: {selector}")
795 | locator = self.page.locator(selector).first
796 | # check() includes actionability checks (visible, enabled)
797 | locator.check(timeout=self.default_action_timeout)
798 | logger.info(f"Checked element: {selector}")
799 | self._human_like_delay(0.2, 0.5) # Small delay after checking
800 | except PlaywrightTimeoutError as e:
801 | logger.error(f"Timeout ({self.default_action_timeout}ms) waiting for element '{selector}' to be actionable for check.")
802 | # Add screenshot on failure
803 | screenshot_path = f"output/check_timeout_{selector.replace(' ','_').replace(':','_').replace('>','_')[:30]}_{int(time.time())}.png"
804 | self.save_screenshot(screenshot_path)
805 | logger.error(f"Saved screenshot on check timeout to: {screenshot_path}")
806 | raise PlaywrightTimeoutError(f"Timeout trying to check element: '{selector}'. Check visibility and enabled state. Screenshot: {screenshot_path}") from e
807 | except PlaywrightError as e:
808 | logger.error(f"PlaywrightError checking element '{selector}': {e}")
809 | raise PlaywrightError(f"Failed to check element '{selector}': {e}") from e
810 | except Exception as e:
811 | logger.error(f"Unexpected error checking '{selector}': {e}", exc_info=True)
812 | raise PlaywrightError(f"Unexpected error checking element '{selector}': {e}") from e
813 |
814 | def uncheck(self, selector: str):
815 | """Unchecks a checkbox."""
816 | if not self.page:
817 | raise PlaywrightError("Browser not started.")
818 | try:
819 | logger.info(f"Attempting to uncheck element: {selector}")
820 | locator = self.page.locator(selector).first
821 | # uncheck() includes actionability checks
822 | locator.uncheck(timeout=self.default_action_timeout)
823 | logger.info(f"Unchecked element: {selector}")
824 | self._human_like_delay(0.2, 0.5) # Small delay
825 | except PlaywrightTimeoutError as e:
826 | logger.error(f"Timeout ({self.default_action_timeout}ms) waiting for element '{selector}' to be actionable for uncheck.")
827 | screenshot_path = f"output/uncheck_timeout_{selector.replace(' ','_').replace(':','_').replace('>','_')[:30]}_{int(time.time())}.png"
828 | self.save_screenshot(screenshot_path)
829 | logger.error(f"Saved screenshot on uncheck timeout to: {screenshot_path}")
830 | raise PlaywrightTimeoutError(f"Timeout trying to uncheck element: '{selector}'. Screenshot: {screenshot_path}") from e
831 | except PlaywrightError as e:
832 | logger.error(f"PlaywrightError unchecking element '{selector}': {e}")
833 | raise PlaywrightError(f"Failed to uncheck element '{selector}': {e}") from e
834 | except Exception as e:
835 | logger.error(f"Unexpected error unchecking '{selector}': {e}", exc_info=True)
836 | raise PlaywrightError(f"Unexpected error unchecking element '{selector}': {e}") from e
837 |
838 | def take_screenshot(self) -> bytes | None:
839 | """Takes a screenshot of the current page and returns bytes."""
840 | if not self.page:
841 | logger.error("Cannot take screenshot, browser not started.")
842 | return None
843 | try:
844 | screenshot_bytes = self.page.screenshot()
845 | logger.info("Screenshot taken (bytes).")
846 | return screenshot_bytes
847 | except Exception as e:
848 | logger.error(f"Error taking screenshot: {e}", exc_info=True)
849 | return None
850 |
851 | def save_screenshot(self, file_path: str) -> bool:
852 | """Takes a screenshot and saves it to the specified file path."""
853 | if not self.page:
854 | logger.error(f"Cannot save screenshot to {file_path}, browser not started.")
855 | return False
856 | try:
857 | # Ensure directory exists
858 | abs_file_path = os.path.abspath(file_path)
859 | os.makedirs(os.path.dirname(abs_file_path), exist_ok=True)
860 |
861 | self.page.screenshot(path=abs_file_path)
862 | logger.info(f"Screenshot saved to: {abs_file_path}")
863 | return True
864 | except Exception as e:
865 | logger.error(f"Error saving screenshot to {file_path}: {e}", exc_info=True)
866 | return False
867 |
868 | def click(self, selector: str):
869 | """Clicks an element, relying on Playwright's built-in actionability checks."""
870 | if not self.page:
871 | raise PlaywrightError("Browser not started.")
872 | try:
873 | logger.info(f"Attempting to click element: {selector}")
874 | locator = self.page.locator(selector).first #
875 | logger.debug(f"Executing click on locator for '{selector}' (with built-in checks)...")
876 | click_delay = random.uniform(50, 150)
877 |
878 | # Optional: Try hover first
879 | try:
880 | locator.hover(timeout=3000) # Short timeout for hover
881 | self._human_like_delay(0.1, 0.3)
882 | except Exception:
883 | logger.debug(f"Hover failed or timed out for {selector}, proceeding with click.")
884 |
885 | # Perform the click with its own timeout
886 | locator.click(delay=click_delay, timeout=self.default_action_timeout)
887 | logger.info(f"Clicked element: {selector}")
888 | self._human_like_delay(0.5, 1.5) # Post-click delay
889 |
890 | except PlaywrightTimeoutError as e:
891 | # Timeout occurred *during* the click action's internal waits
892 | logger.error(f"Timeout ({self.default_action_timeout}ms) waiting for element '{selector}' to be actionable for click. Element might be obscured, disabled, unstable, or not found.")
893 | # Add more context to the error message
894 | screenshot_path = f"output/click_timeout_{selector.replace(' ','_').replace(':','_').replace('>','_')[:30]}_{int(time.time())}.png"
895 | self.save_screenshot(screenshot_path)
896 | logger.error(f"Saved screenshot on click timeout to: {screenshot_path}")
897 | raise PlaywrightTimeoutError(f"Timeout trying to click element: '{selector}'. Check visibility, interactability, and selector correctness. Screenshot saved to {screenshot_path}") from e
898 | except PlaywrightError as e:
899 | # Other errors during click
900 | logger.error(f"PlaywrightError clicking element '{selector}': {e}")
901 | raise PlaywrightError(f"Failed to click element '{selector}': {e}") from e
902 | except Exception as e:
903 | logger.error(f"Unexpected error clicking '{selector}': {e}", exc_info=True)
904 | raise PlaywrightError(f"Unexpected error clicking element '{selector}': {e}") from e
905 |
906 | def type(self, selector: str, text: str):
907 | """
908 | Inputs text into an element, prioritizing the robust `fill` method.
909 | Includes fallback to `type`.
910 | """
911 | if not self.page:
912 | raise PlaywrightError("Browser not started.")
913 | try:
914 | logger.info(f"Attempting to input text '{text[:30]}...' into element: {selector}")
915 | locator = self.page.locator(selector).first
916 |
917 | # --- Strategy 1: Use fill() ---
918 | # fill() clears the field first and inputs text.
919 | # It performs actionability checks (visible, enabled, editable etc.)
920 |
921 | logger.debug(f"Trying to 'fill' locator for '{selector}' (includes actionability checks)...")
922 | try:
923 | if not self.headless: time.sleep(0.2)
924 | locator.fill(text, timeout=self.default_action_timeout) # Use default action timeout
925 | logger.info(f"'fill' successful for element: {selector}")
926 | self._human_like_delay(0.3, 0.8) # Delay after successful input
927 | return # Success! Exit the method.
928 | except (PlaywrightTimeoutError, PlaywrightError) as fill_error:
929 | logger.warning(f"'fill' action failed for '{selector}': {fill_error}. Attempting fallback to 'type'.")
930 |
931 | # Proceed to fallback
932 |
933 | # --- Strategy 2: Fallback to type() ---
934 | logger.debug(f"Trying fallback 'type' for locator '{selector}'...")
935 | try:
936 | # Ensure element is clear before typing as a fallback precaution
937 | locator.clear(timeout=self.default_action_timeout * 0.5) # Quick clear attempt
938 | self._human_like_delay(0.1, 0.3)
939 | typing_delay_ms = random.uniform(90, 180)
940 | locator.type(text, delay=typing_delay_ms, timeout=self.default_action_timeout)
941 | logger.info(f"Fallback 'type' successful for element: {selector}")
942 | self._human_like_delay(0.3, 0.8)
943 | return # Success!
944 | except (PlaywrightTimeoutError, PlaywrightError) as type_error:
945 | logger.error(f"Both 'fill' and fallback 'type' failed for '{selector}'. Last error ('type'): {type_error}")
946 | # Raise the error from the 'type' attempt as it was the last one tried
947 | screenshot_path = f"output/type_fail_{selector.replace(' ','_').replace(':','_').replace('>','_')[:30]}_{int(time.time())}.png"
948 | self.save_screenshot(screenshot_path)
949 | logger.error(f"Saved screenshot on type failure to: {screenshot_path}")
950 | # Raise a combined error or the last one
951 | raise PlaywrightError(f"Failed to input text into element '{selector}' using both fill and type. Last error: {type_error}. Screenshot: {screenshot_path}") from type_error
952 |
953 | # Catch errors related to finding/interacting
954 | except PlaywrightTimeoutError as e:
955 | # This might catch timeouts from clear() or the actionability checks within fill/type
956 | logger.error(f"Timeout ({self.default_action_timeout}ms) during input operation stages for selector: '{selector}'. Element might not become actionable.")
957 | screenshot_path = f"output/input_timeout_{selector.replace(' ','_').replace(':','_').replace('>','_')[:30]}_{int(time.time())}.png"
958 | self.save_screenshot(screenshot_path)
959 | logger.error(f"Saved screenshot on input timeout to: {screenshot_path}")
960 | raise PlaywrightTimeoutError(f"Timeout trying to input text into element: '{selector}'. Check interactability. Screenshot: {screenshot_path}") from e
961 | except PlaywrightError as e:
962 | # Covers other Playwright issues like element detached during operation
963 | logger.error(f"PlaywrightError inputting text into element '{selector}': {e}")
964 | raise PlaywrightError(f"Failed to input text into element '{selector}': {e}") from e
965 | except Exception as e:
966 | logger.error(f"Unexpected error inputting text into '{selector}': {e}", exc_info=True)
967 | raise PlaywrightError(f"Unexpected error inputting text into element '{selector}': {e}") from e
968 |
969 | def scroll(self, direction: str):
970 | """Scrolls the page up or down with a slight delay."""
971 | if not self.page:
972 | raise PlaywrightError("Browser not started.")
973 | try:
974 | scroll_amount = "window.innerHeight"
975 | if direction == "down":
976 | self.page.evaluate(f"window.scrollBy(0, {scroll_amount})")
977 | logger.info("Scrolled down.")
978 | elif direction == "up":
979 | self.page.evaluate(f"window.scrollBy(0, -{scroll_amount})")
980 | logger.info("Scrolled up.")
981 | else:
982 | logger.warning(f"Invalid scroll direction: {direction}")
983 | return # Don't delay for invalid direction
984 | self._human_like_delay(0.4, 0.8) # Delay after scrolling
985 | except Exception as e:
986 | logger.error(f"Error scrolling {direction}: {e}", exc_info=True)
987 |
988 | def press(self, selector: str, keys: str):
989 | """Presses key(s) on a specific element."""
990 | if not self.page:
991 | raise PlaywrightError("Browser not started.")
992 | try:
993 | logger.info(f"Attempting to press '{keys}' on element: {selector}")
994 | locator = self._get_locator(selector)
995 | # Ensure element is actionable first (visible, enabled) before pressing
996 | expect(locator).to_be_enabled(timeout=self.default_action_timeout / 2) # Quick check
997 | expect(locator).to_be_visible(timeout=self.default_action_timeout / 2)
998 | locator.press(keys, timeout=self.default_action_timeout)
999 | logger.info(f"Pressed '{keys}' on element: {selector}")
1000 | self._human_like_delay(0.2, 0.6) # Small delay after key press
1001 | except (PlaywrightTimeoutError, PlaywrightError, AssertionError) as e: # Catch expect failures too
1002 | error_msg = f"Timeout or error pressing '{keys}' on element '{selector}': {type(e).__name__} - {e}"
1003 | logger.error(error_msg)
1004 | screenshot_path = f"output/press_fail_{selector.replace(' ','_').replace(':','_').replace('>','_')[:30]}_{int(time.time())}.png"
1005 | self.save_screenshot(screenshot_path)
1006 | logger.error(f"Saved screenshot on press failure to: {screenshot_path}")
1007 | raise PlaywrightError(f"{error_msg}. Screenshot: {screenshot_path}") from e
1008 | except Exception as e:
1009 | logger.error(f"Unexpected error pressing '{keys}' on '{selector}': {e}", exc_info=True)
1010 | raise PlaywrightError(f"Unexpected error pressing '{keys}' on element '{selector}': {e}") from e
1011 |
1012 | def drag_and_drop(self, source_selector: str, target_selector: str):
1013 | """Drags an element defined by source_selector to an element defined by target_selector."""
1014 | if not self.page:
1015 | raise PlaywrightError("Browser not started.")
1016 | try:
1017 | logger.info(f"Attempting to drag '{source_selector}' to '{target_selector}'")
1018 | source_locator = self._get_locator(source_selector)
1019 | target_locator = self._get_locator(target_selector)
1020 |
1021 | # Optional: Check visibility/existence before drag attempt
1022 | expect(source_locator).to_be_visible(timeout=self.default_action_timeout / 2)
1023 | expect(target_locator).to_be_visible(timeout=self.default_action_timeout / 2)
1024 |
1025 | # Perform drag_to with default timeout
1026 | source_locator.drag_to(target_locator, timeout=self.default_action_timeout)
1027 | logger.info(f"Successfully dragged '{source_selector}' to '{target_selector}'")
1028 | self._human_like_delay(0.5, 1.2) # Delay after drag/drop
1029 | except (PlaywrightTimeoutError, PlaywrightError, AssertionError) as e:
1030 | error_msg = f"Timeout or error dragging '{source_selector}' to '{target_selector}': {type(e).__name__} - {e}"
1031 | logger.error(error_msg)
1032 | screenshot_path = f"output/drag_fail_{source_selector.replace(' ','_')[:20]}_{target_selector.replace(' ','_')[:20]}_{int(time.time())}.png"
1033 | self.save_screenshot(screenshot_path)
1034 | logger.error(f"Saved screenshot on drag failure to: {screenshot_path}")
1035 | raise PlaywrightError(f"{error_msg}. Screenshot: {screenshot_path}") from e
1036 | except Exception as e:
1037 | logger.error(f"Unexpected error dragging '{source_selector}' to '{target_selector}': {e}", exc_info=True)
1038 | raise PlaywrightError(f"Unexpected error dragging '{source_selector}' to '{target_selector}': {e}") from e
1039 |
1040 | def wait(self,
1041 | timeout_seconds: Optional[float] = None,
1042 | selector: Optional[str] = None,
1043 | state: Optional[str] = None, # 'visible', 'hidden', 'enabled', 'disabled', 'attached', 'detached'
1044 | url: Optional[str] = None, # String, regex, or function
1045 | ):
1046 | """Performs various types of waits based on provided parameters."""
1047 | if not self.page:
1048 | raise PlaywrightError("Browser not started.")
1049 |
1050 | try:
1051 | if timeout_seconds is not None and selector is None and state is None and url is None:
1052 | # Simple time wait
1053 | logger.info(f"Waiting for {timeout_seconds:.2f} seconds...")
1054 | self.page.wait_for_timeout(timeout_seconds * 1000)
1055 | logger.info(f"Wait finished after {timeout_seconds:.2f} seconds.")
1056 |
1057 | elif selector and state:
1058 | # Wait for element state
1059 | wait_timeout = self.default_action_timeout # Use default action timeout for element waits
1060 | logger.info(f"Waiting for element '{selector}' to be '{state}' (max {wait_timeout}ms)...")
1061 | locator = self._get_locator(selector) # Handles potential errors
1062 | locator.wait_for(state=state, timeout=wait_timeout)
1063 | logger.info(f"Wait finished: Element '{selector}' is now '{state}'.")
1064 |
1065 | elif url:
1066 | # Wait for URL
1067 | wait_timeout = self.default_navigation_timeout # Use navigation timeout for URL waits
1068 | logger.info(f"Waiting for URL matching '{url}' (max {wait_timeout}ms)...")
1069 | self.page.wait_for_url(url, timeout=wait_timeout)
1070 | logger.info(f"Wait finished: URL now matches '{url}'.")
1071 |
1072 | else:
1073 | logger.info(f"Waiting for 5 seconds...")
1074 | self.page.wait_for_timeout(5 * 1000)
1075 | logger.info(f"Wait finished after {5:.2f} seconds.")
1076 |
1077 | # Optional small delay after successful condition wait
1078 | if selector or url:
1079 | self._human_like_delay(0.1, 0.3)
1080 |
1081 | return {"success": True, "message": "Wait condition met successfully."}
1082 |
1083 | except PlaywrightTimeoutError as e:
1084 | error_msg = f"Timeout waiting for condition: {e}"
1085 | logger.error(error_msg)
1086 | # Don't save screenshot for wait timeouts usually, unless specifically needed
1087 | return {"success": False, "message": error_msg}
1088 | except (PlaywrightError, ValueError) as e:
1089 | error_msg = f"Error during wait: {type(e).__name__}: {e}"
1090 | logger.error(error_msg)
1091 | return {"success": False, "message": error_msg}
1092 | except Exception as e:
1093 | error_msg = f"Unexpected error during wait: {e}"
1094 | logger.error(error_msg, exc_info=True)
1095 | return {"success": False, "message": error_msg}
1096 |
1097 |
1098 |
1099 | def start(self):
1100 | """Starts Playwright, launches browser, creates context/page, and attaches console listener."""
1101 | try:
1102 | logger.info("Starting Playwright...")
1103 | self.playwright = sync_playwright().start()
1104 | # Consider adding args for anti-detection if needed:
1105 | browser_args = ['--disable-blink-features=AutomationControlled']
1106 | self.browser = self.playwright.chromium.launch(headless=self.headless, args=browser_args)
1107 | # self.browser = self.playwright.chromium.launch(headless=self.headless)
1108 |
1109 | context_options = self.browser.new_context(
1110 | user_agent=self._get_random_user_agent(),
1111 | viewport=self._get_random_viewport(),
1112 | ignore_https_errors=True,
1113 | java_script_enabled=True,
1114 | extra_http_headers=COMMON_HEADERS,
1115 | )
1116 | context_options = {
1117 | "user_agent": self._get_random_user_agent(),
1118 | "viewport": self._get_random_viewport(),
1119 | "ignore_https_errors": True,
1120 | "java_script_enabled": True,
1121 | "extra_http_headers": COMMON_HEADERS,
1122 | }
1123 |
1124 | loaded_state = False
1125 | if self.auth_state_path and os.path.exists(self.auth_state_path):
1126 | try:
1127 | logger.info(f"Attempting to load authentication state from: {self.auth_state_path}")
1128 | context_options["storage_state"] = self.auth_state_path
1129 | loaded_state = True
1130 | except Exception as e:
1131 | logger.error(f"Failed to load storage state from '{self.auth_state_path}': {e}. Proceeding without saved state.", exc_info=True)
1132 | # Remove the invalid option if loading failed
1133 | if "storage_state" in context_options:
1134 | del context_options["storage_state"]
1135 | elif self.auth_state_path:
1136 | logger.warning(f"Authentication state file not found at '{self.auth_state_path}'. Proceeding without saved state. Run generation script if needed.")
1137 | else:
1138 | logger.info("No authentication state path provided. Proceeding without saved state.")
1139 |
1140 | self.context = self.browser.new_context(**context_options)
1141 |
1142 | self.context.set_default_navigation_timeout(self.default_navigation_timeout)
1143 | self.context.set_default_timeout(self.default_action_timeout)
1144 | self.context.add_init_script(HIDE_WEBDRIVER_SCRIPT)
1145 |
1146 | self.page = self.context.new_page()
1147 |
1148 | # Initialize DomService with the created page
1149 | self._dom_service = DomService(self.page) # Instantiate here
1150 |
1151 | # --- Attach Console Listener ---
1152 | self.page.on('console', self._handle_console_message)
1153 | logger.info("Attached console message listener.")
1154 | self.page.on('response', self._handle_response) # <<< Attach network listener
1155 | logger.info("Attached network response listener.")
1156 | self.page.on('requestfailed', self._handle_request_failed)
1157 | logger.info("Attached network failed listener.")
1158 | self.panel.inject_recorder_ui_scripts() # inject recorder ui
1159 |
1160 | # -----------------------------
1161 | logger.info("Browser context and page created.")
1162 |
1163 | except Exception as e:
1164 | logger.error(f"Failed to start Playwright or launch browser: {e}", exc_info=True)
1165 | self.close() # Ensure cleanup on failure
1166 | raise
1167 |
1168 | def close(self):
1169 | """Closes the browser and stops Playwright."""
1170 | self.panel.remove_recorder_panel()
1171 | self.remove_click_listener()
1172 | try:
1173 | if self.page and not self.page.is_closed():
1174 | try:
1175 | self.page.remove_listener('response', self._handle_response) # <<< Remove network listener
1176 | logger.debug("Removed network response listener.")
1177 | except Exception as e: logger.warning(f"Could not remove response listener: {e}")
1178 | try:
1179 | self.page.remove_listener('console', self._handle_console_message)
1180 | logger.debug("Removed console message listener.")
1181 | except Exception as e: logger.warning(f"Could not remove console listener: {e}")
1182 | try:
1183 | self.page.remove_listener('requestfailed', self._handle_request_failed) # <<< Remove requestfailed listener
1184 | logger.debug("Removed network requestfailed listener.")
1185 | except Exception as e: logger.warning(f"Could not remove requestfailed listener: {e}")
1186 | self._dom_service = None
1187 | if self.page and not self.page.is_closed():
1188 | # logger.debug("Closing page...") # Added for clarity
1189 | self.page.close()
1190 | # logger.debug("Page closed.")
1191 | else:
1192 | logger.debug("Page already closed or not initialized.")
1193 | if self.context:
1194 | self.context.close()
1195 | logger.info("Browser context closed.")
1196 | if self.browser:
1197 | self.browser.close()
1198 | logger.info("Browser closed.")
1199 | if self.playwright:
1200 | self.playwright.stop()
1201 | logger.info("Playwright stopped.")
1202 | except Exception as e:
1203 | logger.error(f"Error during browser/Playwright cleanup: {e}", exc_info=True)
1204 | finally:
1205 | self.page = None
1206 | self.context = None
1207 | self.browser = None
1208 | self.playwright = None
1209 | self.console_messages = [] # Clear messages on final close
1210 | self.network_requests = [] # Clear network data on final close
1211 | self._recorder_ui_injected = False
1212 |
```
--------------------------------------------------------------------------------
/src/execution/executor.py:
--------------------------------------------------------------------------------
```python
1 | # /src/executor.py
2 | import json
3 | import logging
4 | import time
5 | import os
6 | from patchright.sync_api import sync_playwright, Page, TimeoutError as PlaywrightTimeoutError, Error as PlaywrightError, expect
7 | from typing import Optional, Dict, Any, Tuple, List
8 | from pydantic import BaseModel, Field
9 | import re
10 | from PIL import Image
11 | from pixelmatch.contrib.PIL import pixelmatch
12 | import io
13 |
14 | from ..browser.browser_controller import BrowserController # Re-use for browser setup/teardown
15 | from ..llm.llm_client import LLMClient
16 | from ..agents.recorder_agent import WebAgent
17 | from ..utils.image_utils import compare_images
18 |
19 | # Define a short timeout specifically for selector validation during healing
20 | HEALING_SELECTOR_VALIDATION_TIMEOUT_MS = 2000
21 |
22 |
23 | class HealingSelectorSuggestion(BaseModel):
24 | """Schema for the LLM's suggested replacement selector during healing."""
25 | new_selector: Optional[str] = Field(None, description="The best suggested alternative CSS selector based on visual and DOM context, or null if no suitable alternative is found.")
26 | reasoning: str = Field(..., description="Explanation for the suggested selector choice or the reason why healing could not determine a better selector.")
27 |
28 | logger = logging.getLogger(__name__)
29 |
30 | class TestExecutor:
31 | """
32 | Executes a recorded test case from a JSON file deterministically using Playwright.
33 | """
34 |
35 | def __init__(self,
36 | llm_client: Optional[LLMClient],
37 | headless: bool = True,
38 | default_timeout: int = 5000, # Default timeout for actions/assertions
39 | enable_healing: bool = False, # Flag for healing
40 | healing_mode: str = 'soft', # Healing mode ('soft' or 'hard')
41 | healing_retries: int = 1, # Max soft healing attempts per step
42 | baseline_dir: str = "./visual_baselines", # Add baseline dir
43 | pixel_threshold: float = 0.01, # Default 1% pixel difference threshold
44 | get_performance: bool = False,
45 | get_network_requests: bool = False
46 | ):
47 | self.headless = headless
48 | self.default_timeout = default_timeout # Milliseconds
49 | self.llm_client = llm_client
50 | self.browser_controller: Optional[BrowserController] = None
51 | self.page: Optional[Page] = None
52 | self.enable_healing = enable_healing
53 | self.healing_mode = healing_mode
54 | self.healing_retries_per_step = healing_retries
55 | self.healing_attempts_log: List[Dict] = [] # To store healing attempts info
56 | self.get_performance = get_performance
57 | self.get_network_requests = get_network_requests
58 |
59 |
60 | logger.info(f"TestExecutor initialized (headless={headless}, timeout={default_timeout}ms).")
61 | log_message = ""
62 | if self.enable_healing:
63 | log_message += f" with Healing ENABLED (mode={self.healing_mode}, retries={self.healing_retries_per_step})"
64 | if not self.llm_client:
65 | logger.warning("Self-healing enabled, but LLMClient not provided. Healing will not function.")
66 | else:
67 | log_message += f" using LLM provider '{self.llm_client.provider}'."
68 | else:
69 | log_message += "."
70 | logger.info(log_message)
71 |
72 | if not self.llm_client and not headless: # Vision verification needs LLM
73 | logger.warning("TestExecutor initialized without LLMClient. Vision-based assertions ('assert_passed_verification') will fail.")
74 | elif self.llm_client:
75 | logger.info(f"TestExecutor initialized (headless={headless}, timeout={default_timeout}ms) with LLMClient for provider '{self.llm_client.provider}'.")
76 | else:
77 | logger.info(f"TestExecutor initialized (headless={headless}, timeout={default_timeout}ms). LLMClient not provided (headless mode or vision assertions not needed).")
78 |
79 | self.baseline_dir = os.path.abspath(baseline_dir)
80 | self.pixel_threshold = pixel_threshold # Store threshold
81 | logger.info(f"TestExecutor initialized (visual baseline dir: {self.baseline_dir}, pixel threshold: {self.pixel_threshold*100:.2f}%)")
82 | os.makedirs(self.baseline_dir, exist_ok=True) # Ensure baseline dir exists
83 |
84 |
85 | def _get_locator(self, selector: str):
86 | """Helper to get a Playwright locator, handling potential errors."""
87 | if not self.page:
88 | raise PlaywrightError("Page is not initialized.")
89 | if not selector:
90 | raise ValueError("Selector cannot be empty.")
91 |
92 | is_likely_xpath = selector.startswith(('/', '(', '//')) or \
93 | ('/' in selector and not any(c in selector for c in ['#', '.', '[', '>', '+', '~']))
94 |
95 | # If it looks like XPath but doesn't have a prefix, add 'css='
96 | # Playwright's locator treats "css=<xpath>" as an XPath selector.
97 | processed_selector = selector
98 | if is_likely_xpath and not selector.startswith(('css=', 'xpath=')):
99 | logger.warning(f"Selector '{selector}' looks like XPath but lacks prefix. Assuming XPath and adding 'css=' prefix.")
100 | processed_selector = f"xpath={selector}"
101 |
102 | try:
103 | logger.debug(f"Attempting to locate using: '{processed_selector}'")
104 | return self.page.locator(processed_selector).first
105 | except Exception as e:
106 | # Catch errors during locator creation itself (e.g., invalid selector syntax)
107 | logger.error(f"Failed to create locator for processed selector: '{processed_selector}'. Original: '{selector}'. Error: {e}")
108 | # Re-raise using the processed selector in the message for clarity
109 | raise PlaywrightError(f"Invalid selector syntax or error creating locator: '{processed_selector}'. Error: {e}") from e
110 |
111 |
112 | def _load_baseline(self, baseline_id: str) -> Tuple[Optional[Image.Image], Optional[Dict]]:
113 | """Loads the baseline image and metadata."""
114 | metadata_path = os.path.join(self.baseline_dir, f"{baseline_id}.json")
115 | image_path = os.path.join(self.baseline_dir, f"{baseline_id}.png") # Assume PNG
116 |
117 | if not os.path.exists(metadata_path) or not os.path.exists(image_path):
118 | logger.error(f"Baseline files not found for ID '{baseline_id}' in {self.baseline_dir}")
119 | return None, None
120 |
121 | try:
122 | with open(metadata_path, 'r', encoding='utf-8') as f:
123 | metadata = json.load(f)
124 | baseline_img = Image.open(image_path).convert("RGBA") # Load and ensure RGBA
125 | logger.info(f"Loaded baseline '{baseline_id}' (Image: {image_path}, Metadata: {metadata_path})")
126 | return baseline_img, metadata
127 | except Exception as e:
128 | logger.error(f"Error loading baseline files for ID '{baseline_id}': {e}", exc_info=True)
129 | return None, None
130 |
131 | def _attempt_soft_healing(
132 | self,
133 | failed_step: Dict[str, Any],
134 | failed_selector: Optional[str],
135 | error_message: str
136 | ) -> Tuple[bool, Optional[str], str]:
137 | """
138 | Attempts to find a new selector using the LLM based on the failed step's context and validate it.
139 |
140 | Returns:
141 | Tuple[bool, Optional[str], str]: (healing_success, new_selector, reasoning)
142 | """
143 | if not self.llm_client:
144 | logger.error("Soft Healing: LLMClient not available.")
145 | return False, None, "LLMClient not configured for healing."
146 | if not self.browser_controller or not self.page:
147 | logger.error("Soft Healing: BrowserController or Page not available.")
148 | return False, None, "Browser state unavailable for healing."
149 |
150 | logger.info(f"Soft Healing: Gathering context for step {failed_step.get('step_id')}")
151 |
152 | try:
153 | current_url = self.browser_controller.get_current_url()
154 | screenshot_bytes = self.browser_controller.take_screenshot()
155 | dom_state = self.browser_controller.get_structured_dom(highlight_all_clickable_elements=False, viewport_expansion=-1)
156 | dom_context_str = "DOM context could not be retrieved."
157 | if dom_state and dom_state.element_tree:
158 | dom_context_str, _ = dom_state.element_tree.generate_llm_context_string(context_purpose='verification')
159 | else:
160 | logger.warning("Soft Healing: Failed to get valid DOM state.")
161 |
162 | if not screenshot_bytes:
163 | logger.error("Soft Healing: Failed to capture screenshot.")
164 | return False, None, "Failed to capture screenshot for context."
165 |
166 | except Exception as e:
167 | logger.error(f"Soft Healing: Error gathering context: {e}", exc_info=True)
168 | return False, None, f"Error gathering context: {e}"
169 |
170 | # Construct the prompt
171 | prompt = f"""You are an AI Test Self-Healing Assistant. A step in an automated test failed, likely due to an incorrect or outdated CSS selector. Your goal is to analyze the current page state and suggest a more robust replacement selector for the intended element.
172 |
173 | **Failed Test Step Information:**
174 | - Step Description: "{failed_step.get('description', 'N/A')}"
175 | - Original Action: "{failed_step.get('action', 'N/A')}"
176 | - Failed Selector: `{failed_selector or 'N/A'}`
177 | - Error Message: "{error_message}"
178 |
179 | **Current Page State:**
180 | - URL: {current_url}
181 | - Attached Screenshot: Analyze the visual layout to identify the target element corresponding to the step description.
182 | - HTML Context (Visible elements, interactive `[index]`, static `(Static)`):
183 | ```html
184 | {dom_context_str}
185 | ```
186 |
187 | **Your Task:**
188 | 1. Based on the step description, the original action, the visual screenshot, AND the HTML context, identify the element the test likely intended to interact with.
189 | 2. Suggest a **single, robust CSS selector** for this element using **NATIVE attributes** (like `id`, `name`, `data-testid`, `data-cy`, `aria-label`, `placeholder`, unique visible text combined with tag, stable class combinations).
190 | 3. **CRITICAL: Do NOT suggest selectors based on `data-ai-id` or unstable attributes (e.g., dynamic classes, complex positional selectors like :nth-child unless absolutely necessary and combined with other stable attributes).**
191 | 4. Prioritize standard, semantic, and test-specific attributes (`id`, `data-testid`, `name`).
192 | 5. If you cannot confidently identify the intended element or find a robust selector, return `null` for `new_selector`.
193 |
194 | **Output Format:** Respond ONLY with a JSON object matching the following schema:
195 | ```json
196 | {{
197 | "new_selector": "YOUR_SUGGESTED_CSS_SELECTOR_OR_NULL",
198 | "reasoning": "Explain your choice of selector, referencing visual cues, HTML attributes, and the original step description. If returning null, explain why."
199 | }}
200 | ```
201 | """
202 |
203 | try:
204 | logger.info("Soft Healing: Requesting selector suggestion from LLM...")
205 | response_obj = self.llm_client.generate_json(
206 | HealingSelectorSuggestion,
207 | prompt,
208 | image_bytes=screenshot_bytes
209 | )
210 |
211 | if isinstance(response_obj, HealingSelectorSuggestion):
212 | if response_obj.new_selector:
213 | suggested_selector = response_obj.new_selector
214 | logger.info(f"Soft Healing: LLM suggested new selector: '{response_obj.new_selector}'. Reasoning: {response_obj.reasoning}")
215 | logger.info(f"Soft Healing: Validating suggested selector '{suggested_selector}'...")
216 | validation_passed = False
217 | validation_reasoning_suffix = ""
218 | try:
219 | # Use page.locator() with a short timeout for existence check
220 | count = self.page.locator(suggested_selector).count()
221 |
222 | if count > 0:
223 | validation_passed = True
224 | logger.info(f"Soft Healing: Validation PASSED. Selector '{suggested_selector}' found {count} element(s).")
225 | if count > 1:
226 | logger.warning(f"Soft Healing: Suggested selector '{suggested_selector}' found {count} elements (expected 1). Will target the first.")
227 | else: # count == 0
228 | logger.warning(f"Soft Healing: Validation FAILED. Selector '{suggested_selector}' found 0 elements within {HEALING_SELECTOR_VALIDATION_TIMEOUT_MS}ms.")
229 | validation_reasoning_suffix = " [Validation Failed: Selector found 0 elements]"
230 |
231 | except PlaywrightTimeoutError:
232 | logger.warning(f"Soft Healing: Validation TIMEOUT ({HEALING_SELECTOR_VALIDATION_TIMEOUT_MS}ms) checking selector '{suggested_selector}'.")
233 | validation_reasoning_suffix = f" [Validation Failed: Timeout after {HEALING_SELECTOR_VALIDATION_TIMEOUT_MS}ms]"
234 | except PlaywrightError as e: # Catch invalid selector syntax errors
235 | logger.warning(f"Soft Healing: Validation FAILED. Invalid selector syntax for '{suggested_selector}'. Error: {e}")
236 | validation_reasoning_suffix = f" [Validation Failed: Invalid selector syntax - {e}]"
237 | except Exception as e:
238 | logger.error(f"Soft Healing: Unexpected error during selector validation for '{suggested_selector}': {e}", exc_info=True)
239 | validation_reasoning_suffix = f" [Validation Error: {type(e).__name__}]"
240 | # --- End Validation Step ---
241 |
242 | # Return success only if validation passed
243 | if validation_passed:
244 | return True, suggested_selector, response_obj.reasoning
245 | else:
246 | # Update reasoning with validation failure details
247 | return False, None, response_obj.reasoning + validation_reasoning_suffix
248 |
249 |
250 | else:
251 | logger.warning(f"Soft Healing: LLM could not suggest a new selector. Reasoning: {response_obj.reasoning}")
252 | return False, None, response_obj.reasoning
253 | elif isinstance(response_obj, str): # LLM returned an error string
254 | logger.error(f"Soft Healing: LLM returned an error: {response_obj}")
255 | return False, None, f"LLM Error: {response_obj}"
256 | else: # Unexpected response type
257 | logger.error(f"Soft Healing: Unexpected response type from LLM: {type(response_obj)}")
258 | return False, None, f"Unexpected LLM response type: {type(response_obj)}"
259 |
260 | except Exception as llm_e:
261 | logger.error(f"Soft Healing: Error during LLM communication: {llm_e}", exc_info=True)
262 | return False, None, f"LLM communication error: {llm_e}"
263 |
264 | def _trigger_hard_healing(self, feature_description: str, original_file_path: str) -> None:
265 | """
266 | Closes the current browser and triggers the WebAgent to re-record the test.
267 | """
268 | logger.warning("--- Triggering Hard Healing (Re-Recording) ---")
269 | if not feature_description:
270 | logger.error("Hard Healing: Cannot re-record without the original feature description.")
271 | return
272 | if not self.llm_client:
273 | logger.error("Hard Healing: Cannot re-record without an LLMClient.")
274 | return
275 |
276 | # 1. Close current browser
277 | try:
278 | if self.browser_controller:
279 | self.browser_controller.close()
280 | self.browser_controller = None
281 | self.page = None
282 | logger.info("Hard Healing: Closed executor browser.")
283 | except Exception as close_err:
284 | logger.error(f"Hard Healing: Error closing executor browser: {close_err}")
285 | # Continue anyway, try to re-record
286 |
287 | # 2. Instantiate Recorder Agent
288 | # NOTE: Assume re-recording is automated. Add flag if interactive needed.
289 | try:
290 | logger.info("Hard Healing: Initializing WebAgent for automated re-recording...")
291 | # Use the existing LLM client
292 | recorder_agent = WebAgent(
293 | llm_client=self.llm_client,
294 | headless=False, # Re-recording needs visible browser initially
295 | is_recorder_mode=True,
296 | automated_mode=True, # Run re-recording automatically
297 | # Pass original filename stem to maybe overwrite or create variant
298 | filename=os.path.splitext(os.path.basename(original_file_path))[0] + "_healed_"
299 | )
300 |
301 | # 3. Run Recorder
302 | logger.info(f"Hard Healing: Starting re-recording for feature: '{feature_description}'")
303 | recording_result = recorder_agent.record(feature_description)
304 |
305 | # 4. Log Outcome
306 | if recording_result.get("success"):
307 | logger.info(f"✅ Hard Healing: Re-recording successful. New test file saved to: {recording_result.get('output_file')}")
308 | else:
309 | logger.error(f"❌ Hard Healing: Re-recording FAILED. Message: {recording_result.get('message')}")
310 |
311 | except Exception as record_err:
312 | logger.critical(f"❌ Hard Healing: Critical error during re-recording setup or execution: {record_err}", exc_info=True)
313 |
314 |
315 | def run_test(self, json_file_path: str) -> Dict[str, Any]:
316 | """Loads and executes the test steps from the JSON file."""
317 | start_time = time.time()
318 | self.healing_attempts_log = [] # Reset log for this run
319 |
320 | any_step_successfully_healed = False
321 |
322 | run_status = {
323 | "test_file": json_file_path,
324 | "status": "FAIL", # Default to fail
325 | "message": "Execution initiated.",
326 | "steps_executed": 0,
327 | "failed_step": None,
328 | "error_details": None,
329 | "screenshot_on_failure": None,
330 | "console_messages_on_failure": [],
331 | "all_console_messages": [],
332 | "performance_timing": None,
333 | "network_requests": [],
334 | "duration_seconds": 0.0,
335 | "healing_enabled": self.enable_healing,
336 | "healing_mode": self.healing_mode if self.enable_healing else "disabled",
337 | "healing_attempts": self.healing_attempts_log, # Reference the list
338 | "healed_file_saved": False,
339 | "healed_steps_count": 0,
340 | "visual_assertion_results": []
341 | }
342 |
343 | try:
344 | # --- Load Test Data ---
345 | logger.info(f"Loading test case from: {json_file_path}")
346 | if not os.path.exists(json_file_path):
347 | raise FileNotFoundError(f"Test file not found: {json_file_path}")
348 | with open(json_file_path, 'r', encoding='utf-8') as f:
349 | test_data = json.load(f)
350 | modified_test_data = test_data.copy()
351 |
352 | steps = modified_test_data.get("steps", [])
353 | viewport = next((json.load(open(os.path.join(self.baseline_dir, f"{step.get('parameters', {}).get('baseline_id')}.json"))).get("viewport_size") for step in steps if step.get("action") == "assert_visual_match" and step.get('parameters', {}).get('baseline_id') and os.path.exists(os.path.join(self.baseline_dir, f"{step.get('parameters', {}).get('baseline_id')}.json"))), None)
354 | test_name = modified_test_data.get("test_name", "Unnamed Test")
355 | feature_description = modified_test_data.get("feature_description", "")
356 | first_navigation_done = False
357 | run_status["test_name"] = test_name
358 | logger.info(f"Executing test: '{test_name}' with {len(steps)} steps.")
359 |
360 | if not steps:
361 | raise ValueError("No steps found in the test file.")
362 |
363 | # --- Setup Browser ---
364 | self.browser_controller = BrowserController(headless=self.headless, viewport_size=viewport)
365 | # Set default timeout before starting the page
366 | self.browser_controller.default_action_timeout = self.default_timeout
367 | self.browser_controller.default_navigation_timeout = max(self.default_timeout, 30000) # Ensure navigation timeout is reasonable
368 | self.browser_controller.start()
369 | self.page = self.browser_controller.page
370 | if not self.page:
371 | raise PlaywrightError("Failed to initialize browser page.")
372 | # Re-apply default timeout to the page context AFTER it's created
373 | self.page.set_default_timeout(self.default_timeout)
374 | logger.info(f"Browser page initialized with default action timeout: {self.default_timeout}ms")
375 |
376 | self.browser_controller.clear_console_messages()
377 | self.browser_controller.clear_network_requests()
378 |
379 | # --- Execute Steps ---
380 | for i, step in enumerate(steps):
381 | step_id = step.get("step_id", i + 1)
382 | action = step.get("action")
383 | original_selector = step.get("selector")
384 | params = step.get("parameters", {})
385 | description = step.get("description", f"Step {step_id}")
386 | wait_after = step.get("wait_after_secs", 0) # Get wait time
387 |
388 | run_status["steps_executed"] = i + 1 # Track steps attempted
389 | logger.info(f"--- Executing Step {step_id}: {action} - {description} ---")
390 | if original_selector: logger.info(f"Original Selector: {original_selector}")
391 | if params: logger.info(f"Parameters: {params}")
392 |
393 | # --- Healing Loop ---
394 | step_healed = False
395 | current_healing_attempts = 0
396 | current_selector = original_selector # Start with the recorded selector
397 | last_error = None # Store the last error encountered
398 | successful_healed_selector_for_step = None
399 | run_status["visual_assertion_results"] = []
400 | while not step_healed and current_healing_attempts <= self.healing_retries_per_step:
401 | try:
402 | if action == "navigate":
403 | url = params.get("url")
404 | if not url: raise ValueError("Missing 'url' parameter for navigate.")
405 | self.browser_controller.goto(url)# Uses default navigation timeout from context
406 | if not first_navigation_done:
407 | if self.get_performance:
408 | run_status["performance_timing"] = self.browser_controller.page_performance_timing
409 | first_navigation_done = True
410 | elif action == "click":
411 | if not current_selector: raise ValueError("Missing 'current_selector' for click.")
412 | locator = self._get_locator(current_selector)
413 | locator.click(timeout=self.default_timeout) # Explicit timeout for action
414 | elif action == "type":
415 | text = params.get("text")
416 | if not current_selector: raise ValueError("Missing 'current_selector' for type.")
417 | if text is None: raise ValueError("Missing 'text' parameter for type.")
418 | locator = self._get_locator(current_selector)
419 | locator.fill(text, timeout=self.default_timeout) # Use fill for robustness
420 | elif action == "scroll": # Less common, but support if recorded
421 | direction = params.get("direction")
422 | if direction not in ["up", "down"]: raise ValueError("Invalid 'direction'.")
423 | amount = "window.innerHeight" if direction=="down" else "-window.innerHeight"
424 | self.page.evaluate(f"window.scrollBy(0, {amount})")
425 | elif action == "check":
426 | if not current_selector: raise ValueError("Missing 'current_selector' for check action.")
427 | # Use the browser_controller method which handles locator/timeout
428 | self.browser_controller.check(current_selector)
429 | elif action == "uncheck":
430 | if not current_selector: raise ValueError("Missing 'current_selector' for uncheck action.")
431 | # Use the browser_controller method
432 | self.browser_controller.uncheck(current_selector)
433 | elif action == "select":
434 | option_label = params.get("option_label")
435 | option_value = params.get("option_value") # Support value too if recorded
436 | option_index_str = params.get("option_index") # Support index if recorded
437 | option_param = None
438 | param_type = None
439 |
440 | if option_label is not None:
441 | option_param = {"label": option_label}
442 | param_type = f"label '{option_label}'"
443 | elif option_value is not None:
444 | option_param = {"value": option_value}
445 | param_type = f"value '{option_value}'"
446 | elif option_index_str is not None and option_index_str.isdigit():
447 | option_param = {"index": int(option_index_str)}
448 | param_type = f"index {option_index_str}"
449 | else:
450 | raise ValueError("Missing 'option_label', 'option_value', or 'option_index' parameter for select action.")
451 |
452 | if not current_selector: raise ValueError("Missing 'current_selector' for select action.")
453 |
454 | logger.info(f"Selecting option by {param_type} in element: {current_selector}")
455 | locator = self._get_locator(current_selector)
456 | locator.select_option(**option_param, timeout=self.default_timeout)
457 | elif action == "wait": # Generic wait action
458 | timeout_s = params.get("timeout_seconds")
459 | target_url = params.get("url")
460 | element_state = params.get("state") # e.g., 'visible', 'hidden'
461 | wait_selector = current_selector # Use current (potentially healed) selector if waiting for element
462 |
463 | if timeout_s is not None and not target_url and not element_state:
464 | # Simple time wait
465 | logger.info(f"Waiting for {timeout_s} seconds...")
466 | self.page.wait_for_timeout(timeout_s * 1000)
467 | elif wait_selector and element_state:
468 | # Wait for element state
469 | logger.info(f"Waiting for element '{wait_selector}' to be '{element_state}' (max {self.default_timeout}ms)...")
470 | locator = self._get_locator(wait_selector)
471 | locator.wait_for(state=element_state, timeout=self.default_timeout)
472 | elif target_url:
473 | # Wait for URL
474 | logger.info(f"Waiting for URL matching '{target_url}' (max {self.browser_controller.default_navigation_timeout}ms)...")
475 | self.page.wait_for_url(target_url, timeout=self.browser_controller.default_navigation_timeout)
476 | else:
477 | raise ValueError("Invalid parameters for 'wait' action. Need timeout_seconds OR (selector and state) OR url.")
478 | elif action == "wait_for_load_state":
479 | state = params.get("state", "load")
480 | self.page.wait_for_load_state(state, timeout=self.browser_controller.default_navigation_timeout) # Use navigation timeout
481 | elif action == "wait_for_selector": # Explicit wait
482 | wait_state = params.get("state", "visible")
483 | timeout = params.get("timeout_ms", self.default_timeout)
484 | if not current_selector: raise ValueError("Missing 'current_selector' for wait_for_selector.")
485 | locator = self._get_locator(current_selector)
486 | locator.wait_for(state=wait_state, timeout=timeout)
487 | elif action == "key_press":
488 | keys = params.get("keys")
489 | if not current_selector: raise ValueError("Missing 'selector' for key_press.")
490 | if not keys: raise ValueError("Missing 'keys' parameter for key_press.")
491 | # Use controller method or locator directly
492 | locator = self._get_locator(current_selector)
493 | locator.press(keys, timeout=self.default_timeout)
494 | # self.browser_controller.press(current_selector, keys) # Alt: if using controller method
495 | elif action == "drag_and_drop":
496 | target_selector = params.get("target_selector")
497 | source_selector = current_selector # Source is in the main 'selector' field
498 | if not source_selector: raise ValueError("Missing source 'selector' for drag_and_drop.")
499 | if not target_selector: raise ValueError("Missing 'target_selector' in parameters for drag_and_drop.")
500 | # Use controller method or locators directly
501 | source_locator = self._get_locator(source_selector)
502 | target_locator = self._get_locator(target_selector)
503 | source_locator.drag_to(target_locator, timeout=self.default_timeout)
504 | # self.browser_controller.drag_and_drop(source_selector, target_selector) # Alt: if using controller
505 |
506 | # --- Assertions ---
507 | elif action == "assert_text_contains":
508 | expected_text = params.get("expected_text")
509 | if not current_selector: raise ValueError("Missing 'current_selector' for assertion.")
510 | if expected_text is None: raise ValueError("Missing 'expected_text'.")
511 | locator = self._get_locator(current_selector)
512 | expect(locator).to_contain_text(expected_text, timeout=self.default_timeout)
513 | elif action == "assert_text_equals":
514 | expected_text = params.get("expected_text")
515 | if not current_selector: raise ValueError("Missing 'current_selector' for assertion.")
516 | if expected_text is None: raise ValueError("Missing 'expected_text'.")
517 | locator = self._get_locator(current_selector)
518 | expect(locator).to_have_text(expected_text, timeout=self.default_timeout)
519 | elif action == "assert_visible":
520 | if not current_selector: raise ValueError("Missing 'current_selector' for assertion.")
521 | locator = self._get_locator(current_selector)
522 | expect(locator).to_be_visible(timeout=self.default_timeout)
523 | elif action == "assert_hidden":
524 | if not current_selector: raise ValueError("Missing 'current_selector' for assertion.")
525 | locator = self._get_locator(current_selector)
526 | expect(locator).to_be_hidden(timeout=self.default_timeout)
527 | elif action == "assert_attribute_equals":
528 | attr_name = params.get("attribute_name")
529 | expected_value = params.get("expected_value")
530 | if not current_selector: raise ValueError("Missing 'current_selector' for assertion.")
531 | if not attr_name: raise ValueError("Missing 'attribute_name'.")
532 | if expected_value is None: raise ValueError("Missing 'expected_value'.")
533 | locator = self._get_locator(current_selector)
534 | expect(locator).to_have_attribute(attr_name, expected_value, timeout=self.default_timeout)
535 | elif action == "assert_element_count":
536 | expected_count = params.get("expected_count")
537 | if not current_selector: raise ValueError("Missing 'current_selector' for assertion.")
538 | if expected_count is None: raise ValueError("Missing 'expected_count'.")
539 | if not isinstance(expected_count, int): raise ValueError("'expected_count' must be an integer.") # Add type check
540 |
541 | # --- FIX: Get locator for count without using .first ---
542 | # Apply the same current_selector processing as in _get_locator if needed
543 | is_likely_xpath = current_selector.startswith(('/', '(', '//')) or \
544 | ('/' in current_selector and not any(c in current_selector for c in ['#', '.', '[', '>', '+', '~']))
545 | processed_selector = current_selector
546 | if is_likely_xpath and not current_selector.startswith(('css=', 'xpath=')):
547 | processed_selector = f"xpath={current_selector}"
548 |
549 | # Get the locator for potentially MULTIPLE elements
550 | count_locator = self.page.locator(processed_selector)
551 | # --- End FIX ---
552 |
553 | logger.info(f"Asserting count of elements matching '{processed_selector}' to be {expected_count}")
554 | expect(count_locator).to_have_count(expected_count, timeout=self.default_timeout)
555 | elif action == "assert_checked":
556 | if not current_selector: raise ValueError("Missing 'current_selector' for assert_checked.")
557 | locator = self._get_locator(current_selector)
558 | # Use Playwright's dedicated assertion for checked state
559 | expect(locator).to_be_checked(timeout=self.default_timeout)
560 | elif action == "assert_not_checked":
561 | if not current_selector: raise ValueError("Missing 'current_selector' for assert_not_checked.")
562 | locator = self._get_locator(current_selector)
563 | # Use .not modifier with the checked assertion
564 | expect(locator).not_to_be_checked(timeout=self.default_timeout)
565 | elif action == "assert_disabled":
566 | if not current_selector: raise ValueError("Missing 'current_selector' for assert_disabled.")
567 | locator = self._get_locator(current_selector)
568 | # Use Playwright's dedicated assertion for disabled state
569 | expect(locator).to_be_disabled(timeout=self.default_timeout)
570 | elif action == "assert_enabled":
571 | if not current_selector: raise ValueError("Missing 'current_selector' for assert_enabled.")
572 | locator = self._get_locator(current_selector)
573 | expect(locator).to_be_enabled(timeout=self.default_timeout)
574 | elif action == "task_replanned":
575 | pass
576 | elif action == "assert_visual_match":
577 | baseline_id = params.get("baseline_id")
578 | element_selector = step.get("selector") # Use step's selector if available
579 | use_llm = params.get("use_llm_fallback", True)
580 | # Allow overriding threshold per step
581 | step_threshold = params.get("pixel_threshold", self.pixel_threshold)
582 |
583 | if not baseline_id:
584 | raise ValueError("Missing 'baseline_id' parameter for assert_visual_match.")
585 |
586 | logger.info(f"--- Performing Visual Assertion: '{baseline_id}' (Selector: {element_selector}, Threshold: {step_threshold*100:.2f}%, LLM: {use_llm}) ---")
587 |
588 | # 1. Load Baseline
589 | baseline_img, baseline_meta = self._load_baseline(baseline_id)
590 | if not baseline_img or not baseline_meta:
591 | raise FileNotFoundError(f"Baseline '{baseline_id}' not found or failed to load.")
592 |
593 | # 2. Capture Current State
594 | current_screenshot_bytes = None
595 | if element_selector:
596 | current_screenshot_bytes = self.browser_controller.take_screenshot_element(element_selector)
597 | else:
598 | current_screenshot_bytes = self.browser_controller.take_screenshot() # Full page
599 |
600 | if not current_screenshot_bytes:
601 | raise PlaywrightError("Failed to capture current screenshot for visual comparison.")
602 |
603 | try:
604 | # Create a BytesIO buffer to treat the bytes like a file
605 | buffer = io.BytesIO(current_screenshot_bytes)
606 | # Open the image from the buffer using Pillow
607 | img = Image.open(buffer)
608 | # Ensure the image is in RGBA format for consistency,
609 | # especially important for pixel comparisons that might expect an alpha channel.
610 | logger.info("received")
611 | current_img = img.convert("RGBA")
612 | except Exception as e:
613 | logger.error(f"Failed to convert bytes to PIL Image: {e}", exc_info=True)
614 | current_img = None
615 |
616 |
617 |
618 | if not current_img:
619 | raise RuntimeError("Failed to process current screenshot bytes into an image.")
620 |
621 |
622 | # 3. Pre-check Dimensions
623 | if baseline_img.size != current_img.size:
624 | size_mismatch_msg = f"Visual Assertion Failed: Image dimensions mismatch for '{baseline_id}'. Baseline: {baseline_img.size}, Current: {current_img.size}."
625 | logger.error(size_mismatch_msg)
626 | # Save current image for debugging
627 | ts = time.strftime("%Y%m%d_%H%M%S")
628 | current_img_path = os.path.join("output", f"visual_fail_{baseline_id}_current_{ts}.png")
629 | current_img.save(current_img_path)
630 | logger.info(f"Saved current image (dimension mismatch) to: {current_img_path}")
631 | raise AssertionError(size_mismatch_msg) # Fail the assertion
632 |
633 | # 4. Pixel Comparison
634 | img_diff = Image.new("RGBA", baseline_img.size) # Image to store diff pixels
635 | try:
636 | mismatched_pixels = pixelmatch(baseline_img, current_img, img_diff, includeAA=True, threshold=0.1) # Use default pixelmatch threshold first
637 | except Exception as pm_error:
638 | logger.error(f"Error during pixelmatch comparison for '{baseline_id}': {pm_error}", exc_info=True)
639 | raise RuntimeError(f"Pixelmatch library error: {pm_error}") from pm_error
640 |
641 |
642 | total_pixels = baseline_img.width * baseline_img.height
643 | diff_ratio = mismatched_pixels / total_pixels if total_pixels > 0 else 0
644 | logger.info(f"Pixel comparison for '{baseline_id}': Mismatched Pixels = {mismatched_pixels}, Total Pixels = {total_pixels}, Difference = {diff_ratio*100:.4f}%")
645 |
646 | # 5. Check against threshold
647 | pixel_match_passed = diff_ratio <= step_threshold
648 | llm_reasoning = None
649 | diff_image_path = None
650 |
651 | if pixel_match_passed:
652 | logger.info(f"✅ Visual Assertion PASSED (Pixel Diff <= Threshold) for '{baseline_id}'.")
653 | # Step completed successfully
654 | else:
655 | logger.warning(f"Visual Assertion: Pixel difference ({diff_ratio*100:.4f}%) exceeds threshold ({step_threshold*100:.2f}%) for '{baseline_id}'.")
656 |
657 | # Save diff image regardless of LLM outcome
658 | ts = time.strftime("%Y%m%d_%H%M%S")
659 | diff_image_path = os.path.join("output", f"visual_diff_{baseline_id}_{ts}.png")
660 | try:
661 | img_diff.save(diff_image_path)
662 | logger.info(f"Saved pixel difference image to: {diff_image_path}")
663 | except Exception as save_err:
664 | logger.error(f"Failed to save diff image: {save_err}")
665 | diff_image_path = None # Mark as failed
666 |
667 | # 6. LLM Fallback
668 | if use_llm and self.llm_client:
669 | logger.info(f"Attempting LLM visual comparison fallback for '{baseline_id}'...")
670 | baseline_bytes = io.BytesIO()
671 | baseline_img.save(baseline_bytes, format='PNG')
672 | baseline_bytes = baseline_bytes.getvalue()
673 |
674 | # --- UPDATED LLM PROMPT for Stitched Image ---
675 | llm_prompt = f"""Analyze the combined image provided below for the purpose of automated software testing.
676 | The LEFT half (labeled '1: Baseline') is the established baseline screenshot.
677 | The RIGHT half (labeled '2: Current') is the current state screenshot.
678 |
679 | Compare these two halves to determine if they are SEMANTICALLY equivalent from a user's perspective.
680 |
681 | IGNORE minor differences like:
682 | - Anti-aliasing variations
683 | - Single-pixel shifts
684 | - Tiny rendering fluctuations
685 | - Small, insignificant dynamic content changes (e.g., blinking cursors, exact timestamps if not the focus).
686 |
687 | FOCUS ON significant differences like:
688 | - Layout changes (elements moved, resized, missing, added)
689 | - Major color changes of key elements
690 | - Text content changes (errors, different labels, etc.)
691 | - Missing or fundamentally different images/icons.
692 |
693 | Baseline ID: "{baseline_id}"
694 | Captured URL (Baseline): "{baseline_meta.get('url_captured', 'N/A')}"
695 | Selector (Baseline): "{baseline_meta.get('selector_captured', 'Full Page')}"
696 |
697 | Based on these criteria, are the two halves (baseline vs. current) functionally and visually equivalent enough to PASS a visual regression test?
698 |
699 | Respond ONLY with "YES" or "NO", followed by a brief explanation justifying your answer by referencing differences between the left and right halves.
700 | Example YES: YES - The left (baseline) and right (current) images are visually equivalent. Minor text rendering differences are ignored.
701 | Example NO: NO - The primary call-to-action button visible on the left (baseline) is missing on the right (current).
702 | """
703 | # --- END UPDATED PROMPT ---
704 |
705 | try:
706 | # No change here, compare_images handles the stitching internally
707 | llm_response = compare_images(llm_prompt, baseline_bytes, current_screenshot_bytes, self.llm_client)
708 | logger.info(f"LLM visual comparison response for '{baseline_id}': {llm_response}")
709 | llm_reasoning = llm_response # Store reasoning
710 |
711 | if llm_response.strip().upper().startswith("YES"):
712 | logger.info(f"✅ Visual Assertion PASSED (LLM Override) for '{baseline_id}'.")
713 | pixel_match_passed = True # Override pixel result
714 | elif llm_response.strip().upper().startswith("NO"):
715 | logger.warning(f"Visual Assertion: LLM confirmed significant difference for '{baseline_id}'.")
716 | pixel_match_passed = False # Confirm failure
717 | else:
718 | logger.warning(f"Visual Assertion: LLM response unclear for '{baseline_id}'. Treating as failure.")
719 | pixel_match_passed = False
720 | except Exception as llm_err:
721 | logger.error(f"LLM visual comparison failed: {llm_err}", exc_info=True)
722 | llm_reasoning = f"LLM Error: {llm_err}"
723 | pixel_match_passed = False # Treat LLM error as failure
724 |
725 | else: # LLM fallback not enabled or LLM not available
726 | logger.warning(f"Visual Assertion: LLM fallback skipped for '{baseline_id}'. Failing based on pixel difference.")
727 | pixel_match_passed = False
728 |
729 | # 7. Handle Final Failure
730 | if not pixel_match_passed:
731 | failure_msg = f"Visual Assertion Failed for '{baseline_id}'. Pixel diff: {diff_ratio*100:.4f}% (Threshold: {step_threshold*100:.2f}%)."
732 | if llm_reasoning: failure_msg += f" LLM Reason: {llm_reasoning}"
733 | logger.error(failure_msg)
734 | # Add details to run_status before raising
735 | visual_failure_details = {
736 | "baseline_id": baseline_id,
737 | "pixel_difference_ratio": diff_ratio,
738 | "pixel_threshold": step_threshold,
739 | "mismatched_pixels": mismatched_pixels,
740 | "diff_image_path": diff_image_path,
741 | "llm_reasoning": llm_reasoning
742 | }
743 | # We need to store this somewhere accessible when raising the final error
744 | # Let's add it directly to the step dict temporarily? Or a dedicated failure context?
745 | # For now, log it and include basics in the AssertionError
746 | run_status["visual_failure_details"] = visual_failure_details # Add to main run status
747 | raise AssertionError(failure_msg) # Fail the step
748 |
749 | visual_result = {
750 | "step_id": step_id,
751 | "baseline_id": baseline_id,
752 | "status": "PASS" if pixel_match_passed else "FAIL",
753 | "pixel_difference_ratio": diff_ratio,
754 | "mismatched_pixels": mismatched_pixels,
755 | "pixel_threshold": step_threshold,
756 | "llm_override": use_llm and not pixel_match_passed and llm_response.strip().upper().startswith("YES") if 'llm_response' in locals() else False,
757 | "llm_reasoning": llm_reasoning,
758 | "diff_image_path": diff_image_path,
759 | "element_selector": element_selector
760 | }
761 | run_status["visual_assertion_results"].append(visual_result)
762 |
763 |
764 | elif action == "assert_passed_verification" or action == "assert_llm_verification":
765 | if not self.llm_client:
766 | raise PlaywrightError("LLMClient not available for vision-based verification step.")
767 | if not description:
768 | raise ValueError("Missing 'description' field for 'assert_passed_verification' step.")
769 | if not self.browser_controller:
770 | raise PlaywrightError("BrowserController not available for state gathering.")
771 |
772 | logger.info("Performing vision-based verification with DOM context...")
773 |
774 | # --- Gather Context ---
775 | screenshot_bytes = self.browser_controller.take_screenshot()
776 | current_url = self.browser_controller.get_current_url()
777 | dom_context_str = "DOM context could not be retrieved." # Default
778 | try:
779 | dom_state = self.browser_controller.get_structured_dom(highlight_all_clickable_elements=False, viewport_expansion=-1) # No highlight during execution verification
780 | if dom_state and dom_state.element_tree:
781 | # Use 'verification' purpose for potentially richer context
782 | dom_context_str, _ = dom_state.element_tree.generate_llm_context_string(context_purpose='verification')
783 | else:
784 | logger.warning("Failed to get valid DOM state for vision verification.")
785 | except Exception as dom_err:
786 | logger.error(f"Error getting DOM context for vision verification: {dom_err}", exc_info=True)
787 | # --------------------
788 |
789 | if not screenshot_bytes:
790 | raise PlaywrightError("Failed to capture screenshot for vision verification.")
791 |
792 |
793 | prompt = f"""Analyze the provided webpage screenshot AND the accompanying HTML context.
794 |
795 | The goal during testing was to verify the following condition: "{description}"
796 | Current URL: {current_url}
797 |
798 | HTML Context (Visible elements, interactive elements marked with `[index]`, static with `(Static)`):
799 | ```html
800 | {dom_context_str}
801 | ```
802 |
803 | Based on BOTH the visual evidence in the screenshot AND the HTML context (Prioritize html context more as screenshot will have some delay from when it was asked and when it was taken), is the verification condition "{description}" currently met?
804 | If you think due to the delay in html AND screenshot, state might have changed from where the condition was met, then also respond with YES
805 |
806 | IMPORTANT: Consider that elements might be in a loading state (e.g., placeholders described) OR a fully loaded state (e.g., actual images shown visually). If the current state reasonably fulfills the ultimate goal implied by the description (even if the exact visual differs due to loading, like placeholders becoming images), respond YES.
807 |
808 | Respond with only "YES" or "NO", followed by a brief explanation justifying your answer using evidence from the screenshot and/or HTML context.
809 | Example Response (Success): YES - The 'Welcome, User!' message [Static id='s15'] is visible in the HTML and visually present at the top of the screenshot.
810 | Example Response (Failure): NO - The HTML context shows an error message element [12] and the screenshot visually confirms the 'Invalid credentials' error.
811 | Example Response (Success - Placeholder Intent): YES - The description asked for 5 placeholders, but the screenshot and HTML show 5 fully loaded images within the expected containers ('div.image-container'). This fulfills the intent of ensuring the 5 image sections are present and populated.
812 | """
813 |
814 |
815 | llm_response = self.llm_client.generate_multimodal(prompt, screenshot_bytes)
816 | logger.debug(f"Vision verification LLM response: {llm_response}")
817 |
818 | if llm_response.strip().upper().startswith("YES"):
819 | logger.info("✅ Vision verification PASSED (with DOM context).")
820 | elif llm_response.strip().upper().startswith("NO"):
821 | logger.error(f"❌ Vision verification FAILED (with DOM context). LLM Reasoning: {llm_response}")
822 | raise AssertionError(f"Vision verification failed: Condition '{description}' not met. LLM Reason: {llm_response}")
823 | elif llm_response.startswith("Error:"):
824 | logger.error(f"❌ Vision verification FAILED due to LLM error: {llm_response}")
825 | raise PlaywrightError(f"Vision verification LLM error: {llm_response}")
826 | else:
827 | logger.error(f"❌ Vision verification FAILED due to unclear LLM response: {llm_response}")
828 | raise AssertionError(f"Vision verification failed: Unclear LLM response. Response: {llm_response}")
829 | # --- Add more actions/assertions as needed ---
830 | else:
831 | logger.warning(f"Unsupported action type '{action}' found in step {step_id}. Skipping.")
832 | # Optionally treat as failure: raise ValueError(f"Unsupported action: {action}")
833 |
834 |
835 | step_healed = True
836 | log_suffix = ""
837 | if current_healing_attempts > 0:
838 | # Store the selector that *worked* (which is current_selector)
839 | successful_healed_selector_for_step = current_selector
840 | log_suffix = f" (Healed after {current_healing_attempts} attempt(s) using selector '{current_selector}')"
841 |
842 | logger.info(f"Step {step_id} completed successfully{log_suffix}.")
843 |
844 |
845 | logger.info(f"Step {step_id} completed successfully.")
846 |
847 | # Optional wait after successful step execution
848 | if wait_after > 0:
849 | logger.debug(f"Waiting for {wait_after}s after step {step_id}...")
850 | time.sleep(wait_after)
851 |
852 | except (PlaywrightError, PlaywrightTimeoutError, ValueError, AssertionError) as e:
853 | # Catch Playwright errors, input errors, and assertion failures (from expect)
854 | last_error = e # Store the error
855 | error_type = type(e).__name__
856 | error_msg = str(e)
857 | logger.warning(f"Attempt {current_healing_attempts + 1} for Step {step_id} failed. Error: {error_type}: {error_msg}")
858 |
859 | # --- Healing Decision Logic ---
860 | is_healable_error = isinstance(e, (PlaywrightTimeoutError, PlaywrightError)) and current_selector is not None
861 | # Refine healable conditions:
862 | # - Timeout finding/interacting with an element
863 | # - Element detached, not visible, not interactable (if selector exists)
864 | # - Exclude navigation errors, value errors from missing params, count mismatches
865 | if isinstance(e, ValueError) or (isinstance(e, AssertionError) and "count" in error_msg.lower()):
866 | is_healable_error = False
867 | if action == "navigate":
868 | is_healable_error = False
869 | if action == "assert_visual_match":
870 | is_healable_error = False
871 |
872 | can_attempt_healing = self.enable_healing and is_healable_error and current_healing_attempts < self.healing_retries_per_step
873 |
874 | if can_attempt_healing:
875 | logger.info(f"Attempting Healing (Mode: {self.healing_mode}) for Step {step_id}...")
876 | healing_success = False
877 | new_selector = None
878 | healing_log_entry = {
879 | "step_id": step_id,
880 | "attempt": current_healing_attempts + 1,
881 | "mode": self.healing_mode,
882 | "success": False,
883 | "original_selector": original_selector,
884 | "failed_selector": current_selector,
885 | "error": f"{error_type}: {error_msg}",
886 | "new_selector": None,
887 | "reasoning": None,
888 | }
889 |
890 | if self.healing_mode == 'soft':
891 | healing_success, new_selector, reasoning = self._attempt_soft_healing(step, current_selector, error_msg)
892 | healing_log_entry["new_selector"] = new_selector
893 | healing_log_entry["reasoning"] = reasoning
894 | if healing_success:
895 | logger.info(f"Soft healing successful for Step {step_id}. New selector: '{new_selector}'")
896 | current_selector = new_selector # Update selector for the next loop iteration
897 | healing_log_entry["success"] = True
898 | else:
899 | logger.warning(f"Soft healing failed for Step {step_id}. Reason: {reasoning}")
900 | # Let the loop proceed to final failure state below
901 |
902 | elif self.healing_mode == 'hard':
903 | logger.warning(f"Hard Healing triggered for Step {step_id} due to error: {error_msg}")
904 | if self.browser_controller:
905 | self.browser_controller.clear_console_messages()
906 | self.browser_controller.clear_network_requests()
907 | healing_log_entry["mode"] = "hard" # Log mode
908 | healing_log_entry["success"] = True # Mark attempt as 'successful' in triggering re-record
909 | self.healing_attempts_log.append(healing_log_entry) # Log before triggering
910 | self._trigger_hard_healing(feature_description, json_file_path)
911 | run_status["status"] = "HEALING_TRIGGERED"
912 | run_status["message"] = f"Hard Healing (re-recording) triggered on Step {step_id}."
913 | run_status["failed_step"] = step # Store the step that triggered it
914 | run_status["error_details"] = f"Hard healing triggered by {error_type}: {error_msg}"
915 | return run_status # Stop execution and return status
916 |
917 | self.healing_attempts_log.append(healing_log_entry) # Log soft healing attempt
918 |
919 | if healing_success:
920 | current_healing_attempts += 1
921 | continue # Go to the next iteration of the while loop to retry with new selector
922 | else:
923 | # Soft healing failed, break the while loop to handle final failure
924 | current_healing_attempts = self.healing_retries_per_step + 1
925 |
926 | else:
927 | # Healing not enabled, max attempts reached, or not a healable error
928 | logger.error(f"❌ Step {step_id} failed permanently. Healing skipped or failed.")
929 | raise last_error # Re-raise the last error to trigger final failure handling
930 |
931 |
932 | # --- End Healing Loop ---
933 |
934 | if successful_healed_selector_for_step:
935 | logger.info(f"Persisting healed selector for Step {step_id}: '{successful_healed_selector_for_step}'")
936 | # Modify the step in the IN-MEMORY list 'steps'
937 | if i < len(steps): # Check index boundary
938 | steps[i]['selector'] = successful_healed_selector_for_step
939 | any_step_successfully_healed = True
940 | run_status["healed_steps_count"] += 1
941 | else:
942 | logger.error(f"Index {i} out of bounds for steps list while persisting healed selector for step {step_id}.")
943 |
944 | # If the while loop finished because max attempts were reached without success
945 | if not step_healed:
946 | logger.error(f"❌ Step {step_id} ('{description}') Failed definitively after {current_healing_attempts} attempt(s).")
947 | run_status["status"] = "FAIL"
948 | run_status["message"] = f"Test failed on step {step_id}: {description}"
949 | run_status["failed_step"] = step
950 | # Use the last captured error
951 | error_type = type(last_error).__name__ if last_error else "UnknownError"
952 | error_msg = str(last_error) if last_error else "Step failed after healing attempts."
953 | run_status["error_details"] = f"{error_type}: {error_msg}"
954 | if run_status["status"] == "FAIL" and step.get("action") == "assert_visual_match" and "visual_failure_details" in run_status:
955 | run_status["error_details"] += f"\nVisual Failure Details: {run_status['visual_failure_details']}"
956 |
957 | # Failure Handling (Screenshot/Logs)
958 | try:
959 | ts = time.strftime("%Y%m%d_%H%M%S")
960 | safe_test_name = re.sub(r'[^\w\-]+', '_', test_name)[:50]
961 | screenshot_path = os.path.join("output", f"failure_{safe_test_name}_step{step_id}_{ts}.png")
962 | if self.browser_controller and self.browser_controller.save_screenshot(screenshot_path):
963 | run_status["screenshot_on_failure"] = screenshot_path
964 | logger.info(f"Failure screenshot saved to: {screenshot_path}")
965 | if self.browser_controller:
966 | run_status["all_console_messages"] = self.browser_controller.get_console_messages()
967 | run_status["console_messages_on_failure"] = [
968 | msg for msg in run_status["all_console_messages"]
969 | if msg['type'] in ['error', 'warning']
970 | ][-5:]
971 | except Exception as fail_handle_e:
972 | logger.error(f"Error during failure handling: {fail_handle_e}")
973 |
974 | # Stop the entire test execution
975 | logger.info("Stopping test execution due to permanent step failure.")
976 | return run_status # Return immediately
977 |
978 | # If loop completes without breaking due to permanent failure
979 | logger.info("--- Setting final status to PASS ---")
980 | run_status["status"] = "PASS"
981 | run_status["message"] = "✅ Test executed successfully."
982 | if any_step_successfully_healed:
983 | run_status["message"] += f" ({run_status['healed_steps_count']} step(s) healed)."
984 | logger.info(run_status["message"])
985 |
986 |
987 | except (FileNotFoundError, ValueError, json.JSONDecodeError) as e:
988 | logger.error(f"Error loading or parsing test file '{json_file_path}': {e}")
989 | run_status["message"] = f"Failed to load/parse test file: {e}"
990 | run_status["error_details"] = f"{type(e).__name__}: {str(e)}"
991 | # status is already FAIL by default
992 | except PlaywrightError as e:
993 | logger.critical(f"A Playwright error occurred during execution: {e}", exc_info=True)
994 | if run_status["error_details"] is None: # If this is the first detailed error
995 | run_status["message"] = f"Playwright error: {str(e)}"
996 | run_status["error_details"] = f"{type(e).__name__}: {str(e)}"
997 | run_status["status"] = "FAIL" # Ensure status is Fail
998 | except Exception as e:
999 | logger.critical(f"An unexpected error occurred during execution: {e}", exc_info=True)
1000 | if run_status["error_details"] is None: # If this is the first detailed error
1001 | run_status["message"] = f"Unexpected execution error: {str(e)}"
1002 | run_status["error_details"] = f"{type(e).__name__}: {str(e)}" # Ensure error_details is set
1003 | run_status["status"] = "FAIL" # Ensure status is Fail
1004 | finally:
1005 | logger.info("--- Ending Test Execution ---")
1006 | if self.browser_controller:
1007 | if self.get_network_requests:
1008 | try: run_status["network_requests"] = self.browser_controller.get_network_requests()
1009 | except: logger.error("Failed to retrieve final network requests.")
1010 | # Performance timing is captured after navigation, check if it exists
1011 | if run_status.get("performance_timing") is None and self.get_performance is not False:
1012 | try: run_status["performance_timing"] = self.browser_controller.get_performance_timing()
1013 | except: logger.error("Failed to retrieve final performance timing.")
1014 | # Console messages captured on failure or here
1015 | if "all_console_messages" not in run_status or not run_status["all_console_messages"]:
1016 | try: run_status["all_console_messages"] = self.browser_controller.get_console_messages()
1017 | except: logger.error("Failed to retrieve final console messages.")
1018 |
1019 | self.browser_controller.close()
1020 | self.browser_controller = None
1021 | self.page = None
1022 |
1023 | end_time = time.time()
1024 | run_status["duration_seconds"] = round(end_time - start_time, 2)
1025 | run_status["healing_attempts"] = self.healing_attempts_log
1026 |
1027 | if any_step_successfully_healed and run_status["status"] != "HEALING_TRIGGERED" and run_status["status"] == "PASS": # Save if healing occurred and not hard-healing
1028 | try:
1029 | logger.info(f"Saving updated test file with {run_status['healed_steps_count']} healed step(s) to: {json_file_path}")
1030 | # modified_test_data should contain the updated steps list
1031 | with open(json_file_path, 'w', encoding='utf-8') as f:
1032 | json.dump(modified_test_data, f, indent=2, ensure_ascii=False)
1033 | run_status["healed_file_saved"] = True
1034 | logger.info(f"Successfully saved healed test file: {json_file_path}")
1035 | # Adjust final message if test passed after healing
1036 | if run_status["status"] == "PASS":
1037 | run_status["message"] = f"✅ Test passed with {run_status['healed_steps_count']} step(s) healed. Updated test file saved."
1038 | except Exception as save_err:
1039 | logger.error(f"Failed to save healed test file '{json_file_path}': {save_err}", exc_info=True)
1040 | run_status["healed_file_saved"] = False
1041 | # Add warning to message if save failed
1042 | if run_status["status"] == "PASS":
1043 | run_status["message"] += " (Warning: Failed to save healed selectors)"
1044 | logger.info(f"Execution finished in {run_status['duration_seconds']:.2f} seconds. Status: {run_status['status']}")
1045 |
1046 | return run_status
1047 |
```