This is page 12 of 12. Use http://codebase.md/cameroncooke/xcodebuildmcp?lines=false&page={x} to view the full context.
# Directory Structure
```
├── .axe-version
├── .claude
│ └── agents
│ └── xcodebuild-mcp-qa-tester.md
├── .cursor
│ ├── BUGBOT.md
│ └── environment.json
├── .cursorrules
├── .github
│ ├── FUNDING.yml
│ ├── ISSUE_TEMPLATE
│ │ ├── bug_report.yml
│ │ ├── config.yml
│ │ └── feature_request.yml
│ └── workflows
│ ├── ci.yml
│ ├── README.md
│ ├── release.yml
│ ├── sentry.yml
│ └── stale.yml
├── .gitignore
├── .prettierignore
├── .prettierrc.js
├── .vscode
│ ├── extensions.json
│ ├── launch.json
│ ├── mcp.json
│ ├── settings.json
│ └── tasks.json
├── AGENTS.md
├── banner.png
├── build-plugins
│ ├── plugin-discovery.js
│ ├── plugin-discovery.ts
│ └── tsconfig.json
├── CHANGELOG.md
├── CLAUDE.md
├── CODE_OF_CONDUCT.md
├── docs
│ ├── CONFIGURATION.md
│ ├── DAP_BACKEND_IMPLEMENTATION_PLAN.md
│ ├── DEBUGGING_ARCHITECTURE.md
│ ├── DEMOS.md
│ ├── dev
│ │ ├── ARCHITECTURE.md
│ │ ├── CODE_QUALITY.md
│ │ ├── CONTRIBUTING.md
│ │ ├── ESLINT_TYPE_SAFETY.md
│ │ ├── MANUAL_TESTING.md
│ │ ├── NODEJS_2025.md
│ │ ├── PLUGIN_DEVELOPMENT.md
│ │ ├── README.md
│ │ ├── RELEASE_PROCESS.md
│ │ ├── RELOADEROO_FOR_XCODEBUILDMCP.md
│ │ ├── RELOADEROO_XCODEBUILDMCP_PRIMER.md
│ │ ├── RELOADEROO.md
│ │ ├── session_management_plan.md
│ │ ├── session-aware-migration-todo.md
│ │ ├── SMITHERY.md
│ │ ├── TEST_RUNNER_ENV_IMPLEMENTATION_PLAN.md
│ │ ├── TESTING.md
│ │ └── ZOD_MIGRATION_GUIDE.md
│ ├── DEVICE_CODE_SIGNING.md
│ ├── GETTING_STARTED.md
│ ├── investigations
│ │ ├── issue-154-screenshot-downscaling.md
│ │ ├── issue-163.md
│ │ ├── issue-debugger-attach-stopped.md
│ │ └── issue-describe-ui-empty-after-debugger-resume.md
│ ├── OVERVIEW.md
│ ├── PRIVACY.md
│ ├── README.md
│ ├── SESSION_DEFAULTS.md
│ ├── TOOLS.md
│ └── TROUBLESHOOTING.md
├── eslint.config.js
├── example_projects
│ ├── .vscode
│ │ └── launch.json
│ ├── iOS
│ │ ├── .cursor
│ │ │ └── rules
│ │ │ └── errors.mdc
│ │ ├── .vscode
│ │ │ └── settings.json
│ │ ├── Makefile
│ │ ├── MCPTest
│ │ │ ├── Assets.xcassets
│ │ │ │ ├── AccentColor.colorset
│ │ │ │ │ └── Contents.json
│ │ │ │ ├── AppIcon.appiconset
│ │ │ │ │ └── Contents.json
│ │ │ │ └── Contents.json
│ │ │ ├── ContentView.swift
│ │ │ ├── MCPTestApp.swift
│ │ │ └── Preview Content
│ │ │ └── Preview Assets.xcassets
│ │ │ └── Contents.json
│ │ ├── MCPTest.xcodeproj
│ │ │ ├── project.pbxproj
│ │ │ └── xcshareddata
│ │ │ └── xcschemes
│ │ │ └── MCPTest.xcscheme
│ │ └── MCPTestUITests
│ │ └── MCPTestUITests.swift
│ ├── iOS_Calculator
│ │ ├── .gitignore
│ │ ├── CalculatorApp
│ │ │ ├── Assets.xcassets
│ │ │ │ ├── AccentColor.colorset
│ │ │ │ │ └── Contents.json
│ │ │ │ ├── AppIcon.appiconset
│ │ │ │ │ └── Contents.json
│ │ │ │ └── Contents.json
│ │ │ ├── CalculatorApp.swift
│ │ │ └── CalculatorApp.xctestplan
│ │ ├── CalculatorApp.xcodeproj
│ │ │ ├── project.pbxproj
│ │ │ └── xcshareddata
│ │ │ └── xcschemes
│ │ │ └── CalculatorApp.xcscheme
│ │ ├── CalculatorApp.xcworkspace
│ │ │ └── contents.xcworkspacedata
│ │ ├── CalculatorAppPackage
│ │ │ ├── .gitignore
│ │ │ ├── Package.swift
│ │ │ ├── Sources
│ │ │ │ └── CalculatorAppFeature
│ │ │ │ ├── BackgroundEffect.swift
│ │ │ │ ├── CalculatorButton.swift
│ │ │ │ ├── CalculatorDisplay.swift
│ │ │ │ ├── CalculatorInputHandler.swift
│ │ │ │ ├── CalculatorService.swift
│ │ │ │ └── ContentView.swift
│ │ │ └── Tests
│ │ │ └── CalculatorAppFeatureTests
│ │ │ └── CalculatorServiceTests.swift
│ │ ├── CalculatorAppTests
│ │ │ └── CalculatorAppTests.swift
│ │ └── Config
│ │ ├── Debug.xcconfig
│ │ ├── Release.xcconfig
│ │ ├── Shared.xcconfig
│ │ └── Tests.xcconfig
│ ├── macOS
│ │ ├── MCPTest
│ │ │ ├── Assets.xcassets
│ │ │ │ ├── AccentColor.colorset
│ │ │ │ │ └── Contents.json
│ │ │ │ ├── AppIcon.appiconset
│ │ │ │ │ └── Contents.json
│ │ │ │ └── Contents.json
│ │ │ ├── ContentView.swift
│ │ │ ├── MCPTest.entitlements
│ │ │ ├── MCPTestApp.swift
│ │ │ └── Preview Content
│ │ │ └── Preview Assets.xcassets
│ │ │ └── Contents.json
│ │ ├── MCPTest.xcodeproj
│ │ │ ├── project.pbxproj
│ │ │ └── xcshareddata
│ │ │ └── xcschemes
│ │ │ └── MCPTest.xcscheme
│ │ └── MCPTestTests
│ │ └── MCPTestTests.swift
│ └── spm
│ ├── .gitignore
│ ├── Package.resolved
│ ├── Package.swift
│ ├── Sources
│ │ ├── long-server
│ │ │ └── main.swift
│ │ ├── quick-task
│ │ │ └── main.swift
│ │ ├── spm
│ │ │ └── main.swift
│ │ └── TestLib
│ │ └── TaskManager.swift
│ └── Tests
│ └── TestLibTests
│ └── SimpleTests.swift
├── LICENSE
├── mcp-install-dark.png
├── package-lock.json
├── package.json
├── README.md
├── scripts
│ ├── analysis
│ │ └── tools-analysis.ts
│ ├── bundle-axe.sh
│ ├── check-code-patterns.js
│ ├── generate-loaders.ts
│ ├── generate-version.ts
│ ├── release.sh
│ ├── tools-cli.ts
│ ├── update-tools-docs.ts
│ └── verify-smithery-bundle.sh
├── server.json
├── smithery.config.js
├── smithery.yaml
├── src
│ ├── core
│ │ ├── __tests__
│ │ │ └── resources.test.ts
│ │ ├── generated-plugins.ts
│ │ ├── generated-resources.ts
│ │ ├── plugin-registry.ts
│ │ ├── plugin-types.ts
│ │ └── resources.ts
│ ├── doctor-cli.ts
│ ├── index.ts
│ ├── mcp
│ │ ├── resources
│ │ │ ├── __tests__
│ │ │ │ ├── devices.test.ts
│ │ │ │ ├── doctor.test.ts
│ │ │ │ ├── session-status.test.ts
│ │ │ │ └── simulators.test.ts
│ │ │ ├── devices.ts
│ │ │ ├── doctor.ts
│ │ │ ├── session-status.ts
│ │ │ └── simulators.ts
│ │ └── tools
│ │ ├── debugging
│ │ │ ├── debug_attach_sim.ts
│ │ │ ├── debug_breakpoint_add.ts
│ │ │ ├── debug_breakpoint_remove.ts
│ │ │ ├── debug_continue.ts
│ │ │ ├── debug_detach.ts
│ │ │ ├── debug_lldb_command.ts
│ │ │ ├── debug_stack.ts
│ │ │ ├── debug_variables.ts
│ │ │ └── index.ts
│ │ ├── device
│ │ │ ├── __tests__
│ │ │ │ ├── build_device.test.ts
│ │ │ │ ├── get_device_app_path.test.ts
│ │ │ │ ├── index.test.ts
│ │ │ │ ├── install_app_device.test.ts
│ │ │ │ ├── launch_app_device.test.ts
│ │ │ │ ├── list_devices.test.ts
│ │ │ │ ├── re-exports.test.ts
│ │ │ │ ├── stop_app_device.test.ts
│ │ │ │ └── test_device.test.ts
│ │ │ ├── build_device.ts
│ │ │ ├── clean.ts
│ │ │ ├── discover_projs.ts
│ │ │ ├── get_app_bundle_id.ts
│ │ │ ├── get_device_app_path.ts
│ │ │ ├── index.ts
│ │ │ ├── install_app_device.ts
│ │ │ ├── launch_app_device.ts
│ │ │ ├── list_devices.ts
│ │ │ ├── list_schemes.ts
│ │ │ ├── show_build_settings.ts
│ │ │ ├── start_device_log_cap.ts
│ │ │ ├── stop_app_device.ts
│ │ │ ├── stop_device_log_cap.ts
│ │ │ └── test_device.ts
│ │ ├── doctor
│ │ │ ├── __tests__
│ │ │ │ ├── doctor.test.ts
│ │ │ │ └── index.test.ts
│ │ │ ├── doctor.ts
│ │ │ ├── index.ts
│ │ │ └── lib
│ │ │ └── doctor.deps.ts
│ │ ├── logging
│ │ │ ├── __tests__
│ │ │ │ ├── index.test.ts
│ │ │ │ ├── start_device_log_cap.test.ts
│ │ │ │ ├── start_sim_log_cap.test.ts
│ │ │ │ ├── stop_device_log_cap.test.ts
│ │ │ │ └── stop_sim_log_cap.test.ts
│ │ │ ├── index.ts
│ │ │ ├── start_device_log_cap.ts
│ │ │ ├── start_sim_log_cap.ts
│ │ │ ├── stop_device_log_cap.ts
│ │ │ └── stop_sim_log_cap.ts
│ │ ├── macos
│ │ │ ├── __tests__
│ │ │ │ ├── build_macos.test.ts
│ │ │ │ ├── build_run_macos.test.ts
│ │ │ │ ├── get_mac_app_path.test.ts
│ │ │ │ ├── index.test.ts
│ │ │ │ ├── launch_mac_app.test.ts
│ │ │ │ ├── re-exports.test.ts
│ │ │ │ ├── stop_mac_app.test.ts
│ │ │ │ └── test_macos.test.ts
│ │ │ ├── build_macos.ts
│ │ │ ├── build_run_macos.ts
│ │ │ ├── clean.ts
│ │ │ ├── discover_projs.ts
│ │ │ ├── get_mac_app_path.ts
│ │ │ ├── get_mac_bundle_id.ts
│ │ │ ├── index.ts
│ │ │ ├── launch_mac_app.ts
│ │ │ ├── list_schemes.ts
│ │ │ ├── show_build_settings.ts
│ │ │ ├── stop_mac_app.ts
│ │ │ └── test_macos.ts
│ │ ├── project-discovery
│ │ │ ├── __tests__
│ │ │ │ ├── discover_projs.test.ts
│ │ │ │ ├── get_app_bundle_id.test.ts
│ │ │ │ ├── get_mac_bundle_id.test.ts
│ │ │ │ ├── index.test.ts
│ │ │ │ ├── list_schemes.test.ts
│ │ │ │ └── show_build_settings.test.ts
│ │ │ ├── discover_projs.ts
│ │ │ ├── get_app_bundle_id.ts
│ │ │ ├── get_mac_bundle_id.ts
│ │ │ ├── index.ts
│ │ │ ├── list_schemes.ts
│ │ │ └── show_build_settings.ts
│ │ ├── project-scaffolding
│ │ │ ├── __tests__
│ │ │ │ ├── index.test.ts
│ │ │ │ ├── scaffold_ios_project.test.ts
│ │ │ │ └── scaffold_macos_project.test.ts
│ │ │ ├── index.ts
│ │ │ ├── scaffold_ios_project.ts
│ │ │ └── scaffold_macos_project.ts
│ │ ├── session-management
│ │ │ ├── __tests__
│ │ │ │ ├── index.test.ts
│ │ │ │ ├── session_clear_defaults.test.ts
│ │ │ │ ├── session_set_defaults.test.ts
│ │ │ │ └── session_show_defaults.test.ts
│ │ │ ├── index.ts
│ │ │ ├── session_clear_defaults.ts
│ │ │ ├── session_set_defaults.ts
│ │ │ └── session_show_defaults.ts
│ │ ├── simulator
│ │ │ ├── __tests__
│ │ │ │ ├── boot_sim.test.ts
│ │ │ │ ├── build_run_sim.test.ts
│ │ │ │ ├── build_sim.test.ts
│ │ │ │ ├── get_sim_app_path.test.ts
│ │ │ │ ├── index.test.ts
│ │ │ │ ├── install_app_sim.test.ts
│ │ │ │ ├── launch_app_logs_sim.test.ts
│ │ │ │ ├── launch_app_sim.test.ts
│ │ │ │ ├── list_sims.test.ts
│ │ │ │ ├── open_sim.test.ts
│ │ │ │ ├── record_sim_video.test.ts
│ │ │ │ ├── screenshot.test.ts
│ │ │ │ ├── stop_app_sim.test.ts
│ │ │ │ └── test_sim.test.ts
│ │ │ ├── boot_sim.ts
│ │ │ ├── build_run_sim.ts
│ │ │ ├── build_sim.ts
│ │ │ ├── clean.ts
│ │ │ ├── describe_ui.ts
│ │ │ ├── discover_projs.ts
│ │ │ ├── get_app_bundle_id.ts
│ │ │ ├── get_sim_app_path.ts
│ │ │ ├── index.ts
│ │ │ ├── install_app_sim.ts
│ │ │ ├── launch_app_logs_sim.ts
│ │ │ ├── launch_app_sim.ts
│ │ │ ├── list_schemes.ts
│ │ │ ├── list_sims.ts
│ │ │ ├── open_sim.ts
│ │ │ ├── record_sim_video.ts
│ │ │ ├── screenshot.ts
│ │ │ ├── show_build_settings.ts
│ │ │ ├── stop_app_sim.ts
│ │ │ └── test_sim.ts
│ │ ├── simulator-management
│ │ │ ├── __tests__
│ │ │ │ ├── erase_sims.test.ts
│ │ │ │ ├── index.test.ts
│ │ │ │ ├── reset_sim_location.test.ts
│ │ │ │ ├── set_sim_appearance.test.ts
│ │ │ │ ├── set_sim_location.test.ts
│ │ │ │ └── sim_statusbar.test.ts
│ │ │ ├── boot_sim.ts
│ │ │ ├── erase_sims.ts
│ │ │ ├── index.ts
│ │ │ ├── list_sims.ts
│ │ │ ├── open_sim.ts
│ │ │ ├── reset_sim_location.ts
│ │ │ ├── set_sim_appearance.ts
│ │ │ ├── set_sim_location.ts
│ │ │ └── sim_statusbar.ts
│ │ ├── swift-package
│ │ │ ├── __tests__
│ │ │ │ ├── active-processes.test.ts
│ │ │ │ ├── index.test.ts
│ │ │ │ ├── swift_package_build.test.ts
│ │ │ │ ├── swift_package_clean.test.ts
│ │ │ │ ├── swift_package_list.test.ts
│ │ │ │ ├── swift_package_run.test.ts
│ │ │ │ ├── swift_package_stop.test.ts
│ │ │ │ └── swift_package_test.test.ts
│ │ │ ├── active-processes.ts
│ │ │ ├── index.ts
│ │ │ ├── swift_package_build.ts
│ │ │ ├── swift_package_clean.ts
│ │ │ ├── swift_package_list.ts
│ │ │ ├── swift_package_run.ts
│ │ │ ├── swift_package_stop.ts
│ │ │ └── swift_package_test.ts
│ │ ├── ui-testing
│ │ │ ├── __tests__
│ │ │ │ ├── button.test.ts
│ │ │ │ ├── describe_ui.test.ts
│ │ │ │ ├── gesture.test.ts
│ │ │ │ ├── index.test.ts
│ │ │ │ ├── key_press.test.ts
│ │ │ │ ├── key_sequence.test.ts
│ │ │ │ ├── long_press.test.ts
│ │ │ │ ├── screenshot.test.ts
│ │ │ │ ├── swipe.test.ts
│ │ │ │ ├── tap.test.ts
│ │ │ │ ├── touch.test.ts
│ │ │ │ └── type_text.test.ts
│ │ │ ├── button.ts
│ │ │ ├── describe_ui.ts
│ │ │ ├── gesture.ts
│ │ │ ├── index.ts
│ │ │ ├── key_press.ts
│ │ │ ├── key_sequence.ts
│ │ │ ├── long_press.ts
│ │ │ ├── screenshot.ts
│ │ │ ├── swipe.ts
│ │ │ ├── tap.ts
│ │ │ ├── touch.ts
│ │ │ └── type_text.ts
│ │ └── utilities
│ │ ├── __tests__
│ │ │ ├── clean.test.ts
│ │ │ └── index.test.ts
│ │ ├── clean.ts
│ │ └── index.ts
│ ├── server
│ │ ├── bootstrap.ts
│ │ └── server.ts
│ ├── smithery.ts
│ ├── test-utils
│ │ └── mock-executors.ts
│ ├── types
│ │ └── common.ts
│ ├── utils
│ │ ├── __tests__
│ │ │ ├── build-utils-suppress-warnings.test.ts
│ │ │ ├── build-utils.test.ts
│ │ │ ├── debugger-simctl.test.ts
│ │ │ ├── environment.test.ts
│ │ │ ├── session-aware-tool-factory.test.ts
│ │ │ ├── session-store.test.ts
│ │ │ ├── simulator-utils.test.ts
│ │ │ ├── test-runner-env-integration.test.ts
│ │ │ ├── typed-tool-factory.test.ts
│ │ │ └── workflow-selection.test.ts
│ │ ├── axe
│ │ │ └── index.ts
│ │ ├── axe-helpers.ts
│ │ ├── build
│ │ │ └── index.ts
│ │ ├── build-utils.ts
│ │ ├── capabilities.ts
│ │ ├── command.ts
│ │ ├── CommandExecutor.ts
│ │ ├── debugger
│ │ │ ├── __tests__
│ │ │ │ └── debugger-manager-dap.test.ts
│ │ │ ├── backends
│ │ │ │ ├── __tests__
│ │ │ │ │ └── dap-backend.test.ts
│ │ │ │ ├── dap-backend.ts
│ │ │ │ ├── DebuggerBackend.ts
│ │ │ │ └── lldb-cli-backend.ts
│ │ │ ├── dap
│ │ │ │ ├── __tests__
│ │ │ │ │ └── transport-framing.test.ts
│ │ │ │ ├── adapter-discovery.ts
│ │ │ │ ├── transport.ts
│ │ │ │ └── types.ts
│ │ │ ├── debugger-manager.ts
│ │ │ ├── index.ts
│ │ │ ├── simctl.ts
│ │ │ ├── tool-context.ts
│ │ │ ├── types.ts
│ │ │ └── ui-automation-guard.ts
│ │ ├── environment.ts
│ │ ├── errors.ts
│ │ ├── execution
│ │ │ ├── index.ts
│ │ │ └── interactive-process.ts
│ │ ├── FileSystemExecutor.ts
│ │ ├── log_capture.ts
│ │ ├── log-capture
│ │ │ ├── device-log-sessions.ts
│ │ │ └── index.ts
│ │ ├── logger.ts
│ │ ├── logging
│ │ │ └── index.ts
│ │ ├── plugin-registry
│ │ │ └── index.ts
│ │ ├── responses
│ │ │ └── index.ts
│ │ ├── runtime-registry.ts
│ │ ├── schema-helpers.ts
│ │ ├── sentry.ts
│ │ ├── session-status.ts
│ │ ├── session-store.ts
│ │ ├── simulator-utils.ts
│ │ ├── template
│ │ │ └── index.ts
│ │ ├── template-manager.ts
│ │ ├── test
│ │ │ └── index.ts
│ │ ├── test-common.ts
│ │ ├── tool-registry.ts
│ │ ├── typed-tool-factory.ts
│ │ ├── validation
│ │ │ └── index.ts
│ │ ├── validation.ts
│ │ ├── version
│ │ │ └── index.ts
│ │ ├── video_capture.ts
│ │ ├── video-capture
│ │ │ └── index.ts
│ │ ├── workflow-selection.ts
│ │ ├── xcode.ts
│ │ ├── xcodemake
│ │ │ └── index.ts
│ │ └── xcodemake.ts
│ └── version.ts
├── tsconfig.json
├── tsconfig.test.json
├── tsconfig.tests.json
├── tsup.config.ts
├── vitest.config.ts
└── XcodeBuildMCP.code-workspace
```
# Files
--------------------------------------------------------------------------------
/docs/dev/MANUAL_TESTING.md:
--------------------------------------------------------------------------------
```markdown
# XcodeBuildMCP Manual Testing Guidelines
This document provides comprehensive guidelines for manual black-box testing of XcodeBuildMCP using Reloaderoo inspect commands. This is the authoritative guide for validating all tools through the Model Context Protocol interface.
## Table of Contents
1. [Testing Philosophy](#testing-philosophy)
2. [Black Box Testing via Reloaderoo](#black-box-testing-via-reloaderoo)
3. [Testing Psychology & Bias Prevention](#testing-psychology--bias-prevention)
4. [Tool Dependency Graph Testing Strategy](#tool-dependency-graph-testing-strategy)
5. [Prerequisites](#prerequisites)
6. [Step-by-Step Testing Process](#step-by-step-testing-process)
7. [Error Testing](#error-testing)
8. [Testing Report Generation](#testing-report-generation)
9. [Troubleshooting](#troubleshooting)
## Testing Philosophy
### 🚨 CRITICAL: THOROUGHNESS OVER EFFICIENCY - NO SHORTCUTS ALLOWED
**ABSOLUTE PRINCIPLE: EVERY TOOL MUST BE TESTED INDIVIDUALLY**
**🚨 MANDATORY TESTING SCOPE - NO EXCEPTIONS:**
- **EVERY SINGLE TOOL** - All tools must be tested individually, one by one
- **NO REPRESENTATIVE SAMPLING** - Testing similar tools does NOT validate other tools
- **NO PATTERN RECOGNITION SHORTCUTS** - Similar-looking tools may have different behaviors
- **NO EFFICIENCY OPTIMIZATIONS** - Thoroughness is more important than speed
- **NO TIME CONSTRAINTS** - This is a long-running task with no deadline pressure
**❌ FORBIDDEN EFFICIENCY SHORTCUTS:**
- **NEVER** assume testing `build_sim_id_proj` validates `build_sim_name_proj`
- **NEVER** skip tools because they "look similar" to tested ones
- **NEVER** use representative sampling instead of complete coverage
- **NEVER** stop testing due to time concerns or perceived redundancy
- **NEVER** group tools together for batch testing
- **NEVER** make assumptions about untested tools based on tested patterns
**✅ REQUIRED COMPREHENSIVE APPROACH:**
1. **Individual Tool Testing**: Each tool gets its own dedicated test execution
2. **Complete Documentation**: Every tool result must be recorded, regardless of outcome
3. **Systematic Progress**: Use TodoWrite to track every single tool as tested/untested
4. **Failure Documentation**: Test tools that cannot work and mark them as failed/blocked
5. **No Assumptions**: Treat each tool as potentially unique requiring individual validation
**TESTING COMPLETENESS VALIDATION:**
- **Start Count**: Record exact number of tools discovered using `npm run tools`
- **End Count**: Verify same number of tools have been individually tested
- **Missing Tools = Testing Failure**: If any tools remain untested, the testing is incomplete
- **TodoWrite Tracking**: Every tool must appear in todo list and be marked completed
## Black Box Testing via Reloaderoo
### 🚨 CRITICAL: Black Box Testing via Reloaderoo Inspect
**DEFINITION: Black Box Testing**
Black Box Testing means testing ONLY through external interfaces without any knowledge of internal implementation. For XcodeBuildMCP, this means testing exclusively through the Model Context Protocol (MCP) interface using Reloaderoo as the MCP client.
**🚨 MANDATORY: RELOADEROO INSPECT IS THE ONLY ALLOWED TESTING METHOD**
**ABSOLUTE TESTING RULES - NO EXCEPTIONS:**
1. **✅ ONLY ALLOWED: Reloaderoo Inspect Commands**
- `npx reloaderoo@latest inspect call-tool "TOOL_NAME" --params 'JSON' -- node build/index.js`
- `npx reloaderoo@latest inspect list-tools -- node build/index.js`
- `npx reloaderoo@latest inspect read-resource "URI" -- node build/index.js`
- `npx reloaderoo@latest inspect server-info -- node build/index.js`
- `npx reloaderoo@latest inspect ping -- node build/index.js`
2. **❌ COMPLETELY FORBIDDEN ACTIONS:**
- **NEVER** call `mcp__XcodeBuildMCP__tool_name()` functions directly
- **NEVER** use MCP server tools as if they were native functions
- **NEVER** access internal server functionality
- **NEVER** read source code to understand how tools work
- **NEVER** examine implementation files during testing
- **NEVER** diagnose internal server issues or registration problems
- **NEVER** suggest code fixes or implementation changes
3. **🚨 CRITICAL VIOLATION EXAMPLES:**
```typescript
// ❌ FORBIDDEN - Direct MCP tool calls
await mcp__XcodeBuildMCP__list_devices();
await mcp__XcodeBuildMCP__build_sim_id_proj({ ... });
// ❌ FORBIDDEN - Using tools as native functions
const devices = await list_devices();
const result = await doctor();
// ✅ CORRECT - Only through Reloaderoo inspect
npx reloaderoo@latest inspect call-tool "list_devices" --params '{}' -- node build/index.js
npx reloaderoo@latest inspect call-tool "doctor" --params '{}' -- node build/index.js
```
**WHY RELOADEROO INSPECT IS MANDATORY:**
- **Higher Fidelity**: Provides clear input/output visibility for each tool call
- **Real-world Simulation**: Tests exactly how MCP clients interact with the server
- **Interface Validation**: Ensures MCP protocol compliance and proper JSON formatting
- **Black Box Enforcement**: Prevents accidental access to internal implementation details
- **Clean State**: Each tool call runs with a fresh MCP server instance, preventing cross-contamination
**IMPORTANT: STATEFUL TOOL LIMITATIONS**
**Reloaderoo Inspect Behavior:**
Reloaderoo starts a fresh MCP server instance for each individual tool call and terminates it immediately after the response. This ensures:
- ✅ **Clean Testing Environment**: No state contamination between tool calls
- ✅ **Isolated Testing**: Each tool test is independent and repeatable
- ✅ **Real-world Accuracy**: Simulates how most MCP clients interact with servers
**Expected False Negatives:**
Some tools rely on in-memory state within the MCP server and will fail when tested via Reloaderoo inspect. These failures are **expected and acceptable** as false negatives:
- **`swift_package_stop`** - Requires in-memory process tracking from `swift_package_run`
- **`stop_app_device`** - Requires in-memory process tracking from `launch_app_device`
- **`stop_app_sim`** - Requires in-memory process tracking from `launch_app_sim`
- **`stop_device_log_cap`** - Requires in-memory session tracking from `start_device_log_cap`
- **`stop_sim_log_cap`** - Requires in-memory session tracking from `start_sim_log_cap`
- **`stop_mac_app`** - Requires in-memory process tracking from `launch_mac_app`
**Testing Protocol for Stateful Tools:**
1. **Test the tool anyway** - Execute the Reloaderoo inspect command
2. **Expect failure** - Tool will likely fail due to missing state
3. **Mark as false negative** - Document the failure as expected due to stateful limitations
4. **Continue testing** - Do not attempt to fix or investigate the failure
5. **Report as finding** - Note in testing report that stateful tools failed as expected
**COMPLETE COVERAGE REQUIREMENTS:**
- ✅ **Test ALL tools individually** - No exceptions, every tool gets manual verification
- ✅ **Follow dependency graphs** - Test tools in correct order based on data dependencies
- ✅ **Capture key outputs** - Record UUIDs, paths, schemes needed by dependent tools
- ✅ **Test real workflows** - Complete end-to-end workflows from discovery to execution
- ✅ **Use tool-summary.js script** - Accurate tool/resource counting and discovery
- ✅ **Document all observations** - Record exactly what you see via testing
- ✅ **Report discrepancies as findings** - Note unexpected results without investigation
**MANDATORY INDIVIDUAL TOOL TESTING PROTOCOL:**
**Step 1: Create Complete Tool Inventory**
```bash
# Use the official tool summary script to get accurate tool count and list
npm run tools > /tmp/summary_output.txt
TOTAL_TOOLS=$(grep "Tools:" /tmp/summary_output.txt | awk '{print $2}')
echo "TOTAL TOOLS TO TEST: $TOTAL_TOOLS"
# Generate detailed tool list and extract tool names
npm run tools:list > /tmp/tools_detailed.txt
grep "^ • " /tmp/tools_detailed.txt | sed 's/^ • //' > /tmp/tool_names.txt
```
**Step 2: Create TodoWrite Task List for Every Tool**
```bash
# Create individual todo items for each tool discovered
# Use the actual tool count from step 1
# Example for first few tools:
# 1. [ ] Test tool: doctor
# 2. [ ] Test tool: list_devices
# 3. [ ] Test tool: list_sims
# ... (continue for ALL $TOTAL_TOOLS tools)
```
**Step 3: Test Each Tool Individually**
For EVERY tool in the list:
```bash
# Test each tool individually - NO BATCHING
npx reloaderoo@latest inspect call-tool "TOOL_NAME" --params 'APPROPRIATE_PARAMS' -- node build/index.js
# Mark tool as completed in TodoWrite IMMEDIATELY after testing
# Record result (success/failure/blocked) for each tool
```
**Step 4: Validate Complete Coverage**
```bash
# Verify all tools tested
COMPLETED_TOOLS=$(count completed todo items)
if [ $COMPLETED_TOOLS -ne $TOTAL_TOOLS ]; then
echo "ERROR: Testing incomplete. $COMPLETED_TOOLS/$TOTAL_TOOLS tested"
exit 1
fi
```
**CRITICAL: NO TOOL LEFT UNTESTED**
- **Every tool name from the JSON list must be individually tested**
- **Every tool must have a TodoWrite entry that gets marked completed**
- **Tools that fail due to missing parameters should be tested anyway and marked as blocked**
- **Tools that require setup (like running processes) should be tested and documented as requiring dependencies**
- **NO ASSUMPTIONS**: Test tools even if they seem redundant or similar to others
**BLACK BOX TESTING ENFORCEMENT:**
- ✅ **Test only through Reloaderoo MCP interface** - Simulates real-world MCP client usage
- ✅ **Use task lists** - Track progress with TodoWrite tool for every single tool
- ✅ **Tick off each tool** - Mark completed in task list after manual verification
- ✅ **Manual oversight** - Human verification of each tool's input and output
- ❌ **Never examine source code** - No reading implementation files during testing
- ❌ **Never diagnose internal issues** - No investigation of build processes or tool registration
- ❌ **Never suggest implementation fixes** - Report issues as findings, don't solve them
- ❌ **Never use scripts for tool testing** - Each tool must be manually executed and verified
## Testing Psychology & Bias Prevention
**COMMON ANTI-PATTERNS TO AVOID:**
**1. Efficiency Bias (FORBIDDEN)**
- **Symptom**: "These tools look similar, I'll test one to validate the others"
- **Correction**: Every tool is unique and must be tested individually
- **Enforcement**: Count tools at start, verify same count tested at end
**2. Pattern Recognition Override (FORBIDDEN)**
- **Symptom**: "I see the pattern, the rest will work the same way"
- **Correction**: Patterns may hide edge cases, bugs, or different implementations
- **Enforcement**: No assumptions allowed, test every tool regardless of apparent similarity
**3. Time Pressure Shortcuts (FORBIDDEN)**
- **Symptom**: "This is taking too long, let me speed up by sampling"
- **Correction**: This is explicitly a long-running task with no time constraints
- **Enforcement**: Thoroughness is the ONLY priority, efficiency is irrelevant
**4. False Confidence (FORBIDDEN)**
- **Symptom**: "The architecture is solid, so all tools must work"
- **Correction**: Architecture validation does not guarantee individual tool functionality
- **Enforcement**: Test tools to discover actual issues, not to confirm assumptions
**MANDATORY MINDSET:**
- **Every tool is potentially broken** until individually tested
- **Every tool may have unique edge cases** not covered by similar tools
- **Every tool deserves individual attention** regardless of apparent redundancy
- **Testing completion means EVERY tool tested**, not "enough tools to validate patterns"
- **The goal is discovering problems**, not confirming everything works
**TESTING COMPLETENESS CHECKLIST:**
- [ ] Generated complete tool list using `npm run tools:list`
- [ ] Created TodoWrite entry for every single tool
- [ ] Tested every tool individually via Reloaderoo inspect
- [ ] Marked every tool as completed in TodoWrite
- [ ] Verified tool count: tested_count == total_count
- [ ] Documented all results, including failures and blocked tools
- [ ] Created final report covering ALL tools, not just successful ones
## Tool Dependency Graph Testing Strategy
**CRITICAL: Tools must be tested in dependency order:**
1. **Foundation Tools** (provide data for other tools):
- `doctor` - System info
- `list_devices` - Device UUIDs
- `list_sims` - Simulator UUIDs
- `discover_projs` - Project/workspace paths
2. **Discovery Tools** (provide metadata for build tools):
- `list_schemes` - Scheme names
- `show_build_settings` - Build settings
3. **Build Tools** (create artifacts for install tools):
- `build_*` tools - Create app bundles
- `get_*_app_path_*` tools - Locate built app bundles
- `get_*_bundle_id` tools - Extract bundle IDs
4. **Installation Tools** (depend on built artifacts):
- `install_app_*` tools - Install built apps
- `launch_app_*` tools - Launch installed apps
5. **Testing Tools** (depend on projects/schemes):
- `test_*` tools - Run test suites
6. **UI Automation Tools** (depend on running apps):
- `describe_ui`, `screenshot`, `tap`, etc.
**MANDATORY: Record Key Outputs**
Must capture and document these values for dependent tools:
- **Device UUIDs** from `list_devices`
- **Simulator UUIDs** from `list_sims`
- **Project/workspace paths** from `discover_projs`
- **Scheme names** from `list_schems_*`
- **App bundle paths** from `get_*_app_path_*`
- **Bundle IDs** from `get_*_bundle_id`
## Prerequisites
1. **Build the server**: `npm run build`
2. **Install jq**: `brew install jq` (required for JSON parsing)
3. **System Requirements**: macOS with Xcode installed, connected devices/simulators optional
4. **AXe video capture (optional)**: run `npm run bundle:axe` before using `record_sim_video` in local tests (not required for unit tests)
## Step-by-Step Testing Process
**Note**: All tool and resource discovery now uses the official `tool-summary.js` script (available as `npm run tools`, `npm run tools:list`, and `npm run tools:all`) instead of direct reloaderoo calls. This ensures accurate counts and lists without hardcoded values.
### Step 1: Programmatic Discovery and Official Testing Lists
#### Generate Official Tool and Resource Lists using tool-summary.js
```bash
# Use the official tool summary script to get accurate counts and lists
npm run tools > /tmp/summary_output.txt
# Extract tool and resource counts from summary
TOOL_COUNT=$(grep "Tools:" /tmp/summary_output.txt | awk '{print $2}')
RESOURCE_COUNT=$(grep "Resources:" /tmp/summary_output.txt | awk '{print $2}')
echo "Official tool count: $TOOL_COUNT"
echo "Official resource count: $RESOURCE_COUNT"
# Generate detailed tool list for testing checklist
npm run tools:list > /tmp/tools_detailed.txt
# Extract tool names from the detailed output
grep "^ • " /tmp/tools_detailed.txt | sed 's/^ • //' > /tmp/tool_names.txt
echo "Tool names saved to /tmp/tool_names.txt"
# Generate detailed resource list for testing checklist
npm run tools:all > /tmp/tools_and_resources.txt
# Extract resource URIs from the detailed output
sed -n '/📚 Available Resources:/,/✅ Tool summary complete!/p' /tmp/tools_and_resources.txt | grep "^ • " | sed 's/^ • //' | cut -d' ' -f1 > /tmp/resource_uris.txt
echo "Resource URIs saved to /tmp/resource_uris.txt"
```
#### Create Tool Testing Checklist
```bash
# Generate markdown checklist from actual tool list
echo "# Official Tool Testing Checklist" > /tmp/tool_testing_checklist.md
echo "" >> /tmp/tool_testing_checklist.md
echo "Total Tools: $TOOL_COUNT" >> /tmp/tool_testing_checklist.md
echo "" >> /tmp/tool_testing_checklist.md
# Add each tool as unchecked item
while IFS= read -r tool_name; do
echo "- [ ] $tool_name" >> /tmp/tool_testing_checklist.md
done < /tmp/tool_names.txt
echo "Tool testing checklist created at /tmp/tool_testing_checklist.md"
```
#### Create Resource Testing Checklist
```bash
# Generate markdown checklist from actual resource list
echo "# Official Resource Testing Checklist" > /tmp/resource_testing_checklist.md
echo "" >> /tmp/resource_testing_checklist.md
echo "Total Resources: $RESOURCE_COUNT" >> /tmp/resource_testing_checklist.md
echo "" >> /tmp/resource_testing_checklist.md
# Add each resource as unchecked item
while IFS= read -r resource_uri; do
echo "- [ ] $resource_uri" >> /tmp/resource_testing_checklist.md
done < /tmp/resource_uris.txt
echo "Resource testing checklist created at /tmp/resource_testing_checklist.md"
```
### Step 2: Tool Schema Discovery for Parameter Testing
#### Extract Tool Schema Information
```bash
# Get schema for specific tool to understand required parameters
TOOL_NAME="list_devices"
jq --arg tool "$TOOL_NAME" '.tools[] | select(.name == $tool) | .inputSchema' /tmp/tools.json
# Get tool description for usage guidance
jq --arg tool "$TOOL_NAME" '.tools[] | select(.name == $tool) | .description' /tmp/tools.json
# Generate parameter template for tool testing
jq --arg tool "$TOOL_NAME" '.tools[] | select(.name == $tool) | .inputSchema.properties // {}' /tmp/tools.json
```
#### Batch Schema Extraction
```bash
# Create schema reference file for all tools
echo "# Tool Schema Reference" > /tmp/tool_schemas.md
echo "" >> /tmp/tool_schemas.md
while IFS= read -r tool_name; do
echo "## $tool_name" >> /tmp/tool_schemas.md
echo "" >> /tmp/tool_schemas.md
# Get description
description=$(jq -r --arg tool "$tool_name" '.tools[] | select(.name == $tool) | .description' /tmp/tools.json)
echo "**Description:** $description" >> /tmp/tool_schemas.md
echo "" >> /tmp/tool_schemas.md
# Get required parameters
required=$(jq -r --arg tool "$tool_name" '.tools[] | select(.name == $tool) | .inputSchema.required // [] | join(", ")' /tmp/tools.json)
if [ "$required" != "" ]; then
echo "**Required Parameters:** $required" >> /tmp/tool_schemas.md
else
echo "**Required Parameters:** None" >> /tmp/tool_schemas.md
fi
echo "" >> /tmp/tool_schemas.md
# Get all parameters
echo "**All Parameters:**" >> /tmp/tool_schemas.md
jq --arg tool "$tool_name" '.tools[] | select(.name == $tool) | .inputSchema.properties // {} | keys[]' /tmp/tools.json | while read param; do
echo "- $param" >> /tmp/tool_schemas.md
done
echo "" >> /tmp/tool_schemas.md
done < /tmp/tool_names.txt
echo "Tool schema reference created at /tmp/tool_schemas.md"
```
### Step 3: Manual Tool-by-Tool Testing
#### 🚨 CRITICAL: STEP-BY-STEP BLACK BOX TESTING PROCESS
**ABSOLUTE RULE: ALL TESTING MUST BE DONE MANUALLY, ONE TOOL AT A TIME USING RELOADEROO INSPECT**
**SYSTEMATIC TESTING PROCESS:**
1. **Create TodoWrite Task List**
- Add all tools (from `npm run tools` count) to task list before starting
- Mark each tool as "pending" initially
- Update status to "in_progress" when testing begins
- Mark "completed" only after manual verification
2. **Test Each Tool Individually**
- Execute ONLY via `npx reloaderoo@latest inspect call-tool "TOOL_NAME" --params 'JSON' -- node build/index.js`
- Wait for complete response before proceeding to next tool
- Read and verify each tool's output manually
- Record key outputs (UUIDs, paths, schemes) for dependent tools
3. **Manual Verification Requirements**
- ✅ **Read each response** - Manually verify tool output makes sense
- ✅ **Check for errors** - Identify any tool failures or unexpected responses
- ✅ **Record UUIDs/paths** - Save outputs needed for dependent tools
- ✅ **Update task list** - Mark each tool complete after verification
- ✅ **Document issues** - Record any problems found during testing
4. **FORBIDDEN SHORTCUTS:**
- ❌ **NO SCRIPTS** - Scripts hide what's happening and prevent proper verification
- ❌ **NO AUTOMATION** - Every tool call must be manually executed and verified
- ❌ **NO BATCHING** - Cannot test multiple tools simultaneously
- ❌ **NO MCP DIRECT CALLS** - Only Reloaderoo inspect commands allowed
#### Phase 1: Infrastructure Validation
**Manual Commands (execute individually):**
```bash
# Test server connectivity
npx reloaderoo@latest inspect ping -- node build/index.js
# Get server information
npx reloaderoo@latest inspect server-info -- node build/index.js
# Verify tool count manually
npx reloaderoo@latest inspect list-tools -- node build/index.js 2>/dev/null | jq '.tools | length'
# Verify resource count manually
npx reloaderoo@latest inspect list-resources -- node build/index.js 2>/dev/null | jq '.resources | length'
```
#### Phase 2: Resource Testing
```bash
# Test each resource systematically
while IFS= read -r resource_uri; do
echo "Testing resource: $resource_uri"
npx reloaderoo@latest inspect read-resource "$resource_uri" -- node build/index.js 2>/dev/null
echo "---"
done < /tmp/resource_uris.txt
```
#### Phase 3: Foundation Tools (Data Collection)
**CRITICAL: Capture ALL key outputs for dependent tools**
```bash
echo "=== FOUNDATION TOOL TESTING & DATA COLLECTION ==="
# 1. Test doctor (no dependencies)
echo "Testing doctor..."
npx reloaderoo@latest inspect call-tool "doctor" --params '{}' -- node build/index.js 2>/dev/null
# 2. Collect device data
echo "Collecting device UUIDs..."
npx reloaderoo@latest inspect call-tool "list_devices" --params '{}' -- node build/index.js 2>/dev/null > /tmp/devices_output.json
DEVICE_UUIDS=$(jq -r '.content[0].text' /tmp/devices_output.json | grep -E "UDID: [A-F0-9-]+" | sed 's/.*UDID: //' | head -2)
echo "Device UUIDs captured: $DEVICE_UUIDS"
# 3. Collect simulator data
echo "Collecting simulator UUIDs..."
npx reloaderoo@latest inspect call-tool "list_sims" --params '{}' -- node build/index.js 2>/dev/null > /tmp/sims_output.json
SIMULATOR_UUIDS=$(jq -r '.content[0].text' /tmp/sims_output.json | grep -E "\([A-F0-9-]+\)" | sed 's/.*(\([A-F0-9-]*\)).*/\1/' | head -3)
echo "Simulator UUIDs captured: $SIMULATOR_UUIDS"
# 4. Collect project data
echo "Collecting project paths..."
npx reloaderoo@latest inspect call-tool "discover_projs" --params '{"workspaceRoot": "/Volumes/Developer/XcodeBuildMCP"}' -- node build/index.js 2>/dev/null > /tmp/projects_output.json
PROJECT_PATHS=$(jq -r '.content[1].text' /tmp/projects_output.json | grep -E "\.xcodeproj$" | sed 's/.*- //' | head -3)
WORKSPACE_PATHS=$(jq -r '.content[2].text' /tmp/projects_output.json | grep -E "\.xcworkspace$" | sed 's/.*- //' | head -2)
echo "Project paths captured: $PROJECT_PATHS"
echo "Workspace paths captured: $WORKSPACE_PATHS"
# Save key data for dependent tools
echo "$DEVICE_UUIDS" > /tmp/device_uuids.txt
echo "$SIMULATOR_UUIDS" > /tmp/simulator_uuids.txt
echo "$PROJECT_PATHS" > /tmp/project_paths.txt
echo "$WORKSPACE_PATHS" > /tmp/workspace_paths.txt
```
#### Phase 4: Discovery Tools (Metadata Collection)
```bash
echo "=== DISCOVERY TOOL TESTING & METADATA COLLECTION ==="
# Collect schemes for each project
while IFS= read -r project_path; do
if [ -n "$project_path" ]; then
echo "Getting schemes for: $project_path"
npx reloaderoo@latest inspect call-tool "list_schems_proj" --params "{\"projectPath\": \"$project_path\"}" -- node build/index.js 2>/dev/null > /tmp/schemes_$$.json
SCHEMES=$(jq -r '.content[1].text' /tmp/schemes_$$.json 2>/dev/null || echo "NoScheme")
echo "$project_path|$SCHEMES" >> /tmp/project_schemes.txt
echo "Schemes captured for $project_path: $SCHEMES"
fi
done < /tmp/project_paths.txt
# Collect schemes for each workspace
while IFS= read -r workspace_path; do
if [ -n "$workspace_path" ]; then
echo "Getting schemes for: $workspace_path"
npx reloaderoo@latest inspect call-tool "list_schemes" --params "{\"workspacePath\": \"$workspace_path\"}" -- node build/index.js 2>/dev/null > /tmp/ws_schemes_$$.json
SCHEMES=$(jq -r '.content[1].text' /tmp/ws_schemes_$$.json 2>/dev/null || echo "NoScheme")
echo "$workspace_path|$SCHEMES" >> /tmp/workspace_schemes.txt
echo "Schemes captured for $workspace_path: $SCHEMES"
fi
done < /tmp/workspace_paths.txt
```
#### Phase 5: Manual Individual Tool Testing (All Tools)
**CRITICAL: Test every single tool manually, one at a time**
**Manual Testing Process:**
1. **Create task list** with TodoWrite tool for all tools (using count from `npm run tools`)
2. **Test each tool individually** with proper parameters
3. **Mark each tool complete** in task list after manual verification
4. **Record results** and observations for each tool
5. **NO SCRIPTS** - Each command executed manually
**STEP-BY-STEP MANUAL TESTING COMMANDS:**
```bash
# STEP 1: Test foundation tools (no parameters required)
# Execute each command individually, wait for response, verify manually
npx reloaderoo@latest inspect call-tool "doctor" --params '{}' -- node build/index.js
# [Wait for response, read output, mark tool complete in task list]
npx reloaderoo@latest inspect call-tool "list_devices" --params '{}' -- node build/index.js
# [Record device UUIDs from response for dependent tools]
npx reloaderoo@latest inspect call-tool "list_sims" --params '{}' -- node build/index.js
# [Record simulator UUIDs from response for dependent tools]
# STEP 2: Test project discovery (use discovered project paths)
npx reloaderoo@latest inspect call-tool "list_schems_proj" --params '{"projectPath": "/actual/path/from/discover_projs.xcodeproj"}' -- node build/index.js
# [Record scheme names from response for build tools]
# STEP 3: Test workspace tools (use discovered workspace paths)
npx reloaderoo@latest inspect call-tool "list_schemes" --params '{"workspacePath": "/actual/path/from/discover_projs.xcworkspace"}' -- node build/index.js
# [Record scheme names from response for build tools]
# STEP 4: Test simulator tools (use captured simulator UUIDs from step 1)
npx reloaderoo@latest inspect call-tool "boot_sim" --params '{"simulatorUuid": "ACTUAL_UUID_FROM_LIST_SIMS"}' -- node build/index.js
# [Verify simulator boots successfully]
# STEP 5: Test build tools (requires project + scheme + simulator from previous steps)
npx reloaderoo@latest inspect call-tool "build_sim_id_proj" --params '{"projectPath": "/actual/project.xcodeproj", "scheme": "ActualSchemeName", "simulatorId": "ACTUAL_SIMULATOR_UUID"}' -- node build/index.js
# [Verify build succeeds and record app bundle path]
```
**CRITICAL: EACH COMMAND MUST BE:**
1. **Executed individually** - One command at a time, manually typed or pasted
2. **Verified manually** - Read the complete response before continuing
3. **Tracked in task list** - Mark tool complete only after verification
4. **Use real data** - Replace placeholder values with actual captured data
5. **Wait for completion** - Allow each command to finish before proceeding
### TESTING VIOLATIONS AND ENFORCEMENT
**🚨 CRITICAL VIOLATIONS THAT WILL TERMINATE TESTING:**
1. **Direct MCP Tool Usage Violation:**
```typescript
// ❌ IMMEDIATE TERMINATION - Using MCP tools directly
await mcp__XcodeBuildMCP__list_devices();
const result = await list_sims();
```
2. **Script-Based Testing Violation:**
```bash
# ❌ IMMEDIATE TERMINATION - Using scripts to test tools
for tool in $(cat tool_list.txt); do
npx reloaderoo inspect call-tool "$tool" --params '{}' -- node build/index.js
done
```
3. **Batching/Automation Violation:**
```bash
# ❌ IMMEDIATE TERMINATION - Testing multiple tools simultaneously
npx reloaderoo inspect call-tool "list_devices" & npx reloaderoo inspect call-tool "list_sims" &
```
4. **Source Code Examination Violation:**
```typescript
// ❌ IMMEDIATE TERMINATION - Reading implementation during testing
const toolImplementation = await Read('/src/mcp/tools/device-shared/list_devices.ts');
```
**ENFORCEMENT PROCEDURE:**
1. **First Violation**: Immediate correction and restart of testing process
2. **Documentation Update**: Add explicit prohibition to prevent future violations
3. **Method Validation**: Ensure all future testing uses only Reloaderoo inspect commands
4. **Progress Reset**: Restart testing from foundation tools if direct MCP usage detected
**VALID TESTING SEQUENCE EXAMPLE:**
```bash
# ✅ CORRECT - Step-by-step manual execution via Reloaderoo
# Tool 1: Test doctor
npx reloaderoo@latest inspect call-tool "doctor" --params '{}' -- node build/index.js
# [Read response, verify, mark complete in TodoWrite]
# Tool 2: Test list_devices
npx reloaderoo@latest inspect call-tool "list_devices" --params '{}' -- node build/index.js
# [Read response, capture UUIDs, mark complete in TodoWrite]
# Tool 3: Test list_sims
npx reloaderoo@latest inspect call-tool "list_sims" --params '{}' -- node build/index.js
# [Read response, capture UUIDs, mark complete in TodoWrite]
# Tool X: Test stateful tool (expected to fail)
npx reloaderoo@latest inspect call-tool "swift_package_stop" --params '{"pid": 12345}' -- node build/index.js
# [Tool fails as expected - no in-memory state available]
# [Mark as "false negative - stateful tool limitation" in TodoWrite]
# [Continue to next tool without investigation]
# Continue individually for all tools (use count from npm run tools)...
```
**HANDLING STATEFUL TOOL FAILURES:**
```bash
# ✅ CORRECT Response to Expected Stateful Tool Failure
# Tool fails with "No process found" or similar state-related error
# Response: Mark tool as "tested - false negative (stateful)" in task list
# Do NOT attempt to diagnose, fix, or investigate the failure
# Continue immediately to next tool in sequence
```
## Error Testing
```bash
# Test error handling systematically
echo "=== Error Testing ==="
# Test with invalid JSON parameters
echo "Testing invalid parameter types..."
npx reloaderoo@latest inspect call-tool list_schems_proj --params '{"projectPath": 123}' -- node build/index.js 2>/dev/null
# Test with non-existent paths
echo "Testing non-existent paths..."
npx reloaderoo@latest inspect call-tool list_schems_proj --params '{"projectPath": "/nonexistent/path.xcodeproj"}' -- node build/index.js 2>/dev/null
# Test with invalid UUIDs
echo "Testing invalid UUIDs..."
npx reloaderoo@latest inspect call-tool boot_sim --params '{"simulatorUuid": "invalid-uuid"}' -- node build/index.js 2>/dev/null
```
## Testing Report Generation
```bash
# Create comprehensive testing session report
cat > TESTING_SESSION_$(date +%Y-%m-%d).md << EOF
# Manual Testing Session - $(date +%Y-%m-%d)
## Environment
- macOS Version: $(sw_vers -productVersion)
- XcodeBuildMCP Version: $(jq -r '.version' package.json 2>/dev/null || echo "unknown")
- Testing Method: Reloaderoo @latest via npx
## Official Counts (Programmatically Verified)
- Total Tools: $TOOL_COUNT
- Total Resources: $RESOURCE_COUNT
## Test Results
[Document test results here]
## Issues Found
[Document any discrepancies or failures]
## Performance Notes
[Document response times and performance observations]
EOF
echo "Testing session template created: TESTING_SESSION_$(date +%Y-%m-%d).md"
```
### Key Commands Reference
```bash
# Essential testing commands
npx reloaderoo@latest inspect ping -- node build/index.js
npx reloaderoo@latest inspect server-info -- node build/index.js
npx reloaderoo@latest inspect list-tools -- node build/index.js | jq '.tools | length'
npx reloaderoo@latest inspect list-resources -- node build/index.js | jq '.resources | length'
npx reloaderoo@latest inspect call-tool TOOL_NAME --params '{}' -- node build/index.js
npx reloaderoo@latest inspect read-resource "xcodebuildmcp://RESOURCE" -- node build/index.js
# Schema extraction
jq --arg tool "TOOL_NAME" '.tools[] | select(.name == $tool) | .inputSchema' /tmp/tools.json
jq --arg tool "TOOL_NAME" '.tools[] | select(.name == $tool) | .description' /tmp/tools.json
```
## Troubleshooting
### Common Issues
#### 1. Reloaderoo Command Timeouts
**Symptoms**: Commands hang or timeout after extended periods
**Cause**: Server startup issues or MCP protocol communication problems
**Resolution**:
- Verify server builds successfully: `npm run build`
- Test direct server startup: `node build/index.js`
- Check for TypeScript compilation errors
#### 2. Tool Parameter Validation Errors
**Symptoms**: Tools return parameter validation errors
**Cause**: Missing or incorrect required parameters
**Resolution**:
- Check tool schema: `jq --arg tool "TOOL_NAME" '.tools[] | select(.name == $tool) | .inputSchema' /tmp/tools.json`
- Verify parameter types and required fields
- Use captured dependency data (UUIDs, paths, schemes)
#### 3. "No Such Tool" Errors
**Symptoms**: Reloaderoo reports tool not found
**Cause**: Tool name mismatch or server registration issues
**Resolution**:
- Verify tool exists in list: `npx reloaderoo@latest inspect list-tools -- node build/index.js | jq '.tools[].name'`
- Check exact tool name spelling and case sensitivity
- Ensure server built successfully
#### 4. Empty or Malformed Responses
**Symptoms**: Tools return empty responses or JSON parsing errors
**Cause**: Tool implementation issues or server errors
**Resolution**:
- Document as testing finding - do not investigate implementation
- Mark tool as "failed - empty response" in task list
- Continue with next tool in sequence
This systematic approach ensures comprehensive, accurate testing using programmatic discovery and validation of all XcodeBuildMCP functionality through the MCP interface exclusively.
```
--------------------------------------------------------------------------------
/scripts/check-code-patterns.js:
--------------------------------------------------------------------------------
```javascript
#!/usr/bin/env node
/**
* XcodeBuildMCP Code Pattern Violations Checker
*
* Validates that all code files follow XcodeBuildMCP-specific architectural patterns.
* This script focuses on domain-specific rules that ESLint cannot express.
*
* USAGE:
* node scripts/check-code-patterns.js [--pattern=vitest|execsync|handler|handler-testing|all]
* node scripts/check-code-patterns.js --help
*
* ARCHITECTURAL RULES ENFORCED:
* 1. NO vitest mocking patterns (vi.mock, vi.fn, .mockResolvedValue, etc.)
* 2. NO execSync usage in production code (use CommandExecutor instead)
* 3. ONLY dependency injection with createMockExecutor() and createMockFileSystemExecutor()
* 4. NO handler signature violations (handlers must have exact MCP SDK signatures)
* 5. NO handler testing violations (test logic functions, not handlers directly)
*
* For comprehensive code quality documentation, see docs/dev/CODE_QUALITY.md
*
* Note: General code quality rules (TypeScript, ESLint) are handled by other tools.
* This script only enforces XcodeBuildMCP-specific architectural patterns.
*/
import { readFileSync, readdirSync, statSync } from 'fs';
import { join, relative } from 'path';
import { fileURLToPath } from 'url';
import { dirname } from 'path';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const projectRoot = join(__dirname, '..');
// Parse command line arguments
const args = process.argv.slice(2);
const patternFilter = args.find(arg => arg.startsWith('--pattern='))?.split('=')[1] || 'all';
const showHelp = args.includes('--help') || args.includes('-h');
if (showHelp) {
console.log(`
XcodeBuildMCP Code Pattern Violations Checker
USAGE:
node scripts/check-code-patterns.js [options]
OPTIONS:
--pattern=TYPE Check specific pattern type (vitest|execsync|handler|handler-testing|server-typing|all) [default: all]
--help, -h Show this help message
PATTERN TYPES:
vitest Check only vitest mocking violations (vi.mock, vi.fn, etc.)
execsync Check only execSync usage in production code
handler Check only handler signature violations
handler-testing Check only handler testing violations (testing handlers instead of logic functions)
server-typing Check only improper server typing violations (Record<string, unknown> casts)
all Check all pattern violations (default)
Note: General code quality (TypeScript, etc.) is handled by ESLint
EXAMPLES:
node scripts/check-code-patterns.js
node scripts/check-code-patterns.js --pattern=vitest
node scripts/check-code-patterns.js --pattern=handler
node scripts/check-code-patterns.js --pattern=handler-testing
node scripts/check-code-patterns.js --pattern=server-typing
`);
process.exit(0);
}
// Patterns for execSync usage in production code (FORBIDDEN)
// Note: execSync is allowed in test files for mocking, but not in production code
const EXECSYNC_PATTERNS = [
/\bexecSync\s*\(/, // Direct execSync usage
/\bexecSyncFn\s*[=:]/, // execSyncFn parameter or assignment
/^import\s+(?!type\s)[^}]*from\s+['"]child_process['"]/m, // Importing from child_process (except type-only imports)
/^import\s+{[^}]*(?:exec|spawn|execSync)[^}]*}\s+from\s+['"](?:node:)?child_process['"]/m, // Named imports of functions
];
// CRITICAL: ALL VITEST MOCKING PATTERNS ARE COMPLETELY FORBIDDEN
// ONLY dependency injection with approved mock utilities is allowed
const VITEST_GENERIC_PATTERNS = [
/vi\.mock\s*\(/, // vi.mock() - BANNED
/vi\.fn\s*\(/, // vi.fn() - BANNED
/vi\.mocked\s*\(/, // vi.mocked() - BANNED
/vi\.spyOn\s*\(/, // vi.spyOn() - BANNED
/vi\.clearAllMocks\s*\(/, // vi.clearAllMocks() - BANNED
/\.mockResolvedValue/, // .mockResolvedValue - BANNED
/\.mockRejectedValue/, // .mockRejectedValue - BANNED
/\.mockReturnValue/, // .mockReturnValue - BANNED
/\.mockImplementation/, // .mockImplementation - BANNED
/\.mockClear/, // .mockClear - BANNED
/\.mockReset/, // .mockReset - BANNED
/\.mockRestore/, // .mockRestore - BANNED
/\.toHaveBeenCalled/, // .toHaveBeenCalled - BANNED
/\.toHaveBeenCalledWith/, // .toHaveBeenCalledWith - BANNED
/MockedFunction/, // MockedFunction type - BANNED
/as MockedFunction/, // Type casting to MockedFunction - BANNED
/\bexecSync\b/, // execSync usage - BANNED (use executeCommand instead)
/\bexecSyncFn\b/, // execSyncFn usage - BANNED (use executeCommand instead)
];
// APPROVED mock utilities - ONLY these are allowed
const APPROVED_MOCK_PATTERNS = [
/\bcreateMockExecutor\b/,
/\bcreateMockFileSystemExecutor\b/,
/\bcreateNoopExecutor\b/,
/\bcreateNoopFileSystemExecutor\b/,
/\bcreateCommandMatchingMockExecutor\b/,
/\bcreateMockEnvironmentDetector\b/,
];
// REFINED PATTERNS - Only flag ACTUAL vitest violations, not approved dependency injection patterns
// Manual executors and mock objects are APPROVED when used for dependency injection
const UNAPPROVED_MOCK_PATTERNS = [
// ONLY ACTUAL VITEST PATTERNS (vi.* usage) - Everything else is approved
/\bmock[A-Z][a-zA-Z0-9]*\s*=\s*vi\./, // mockSomething = vi.fn() - vitest assignments only
// No other patterns - manual executors and mock objects are approved for dependency injection
];
// Function to check if a line contains unapproved mock patterns
function hasUnapprovedMockPattern(line) {
// Skip lines that contain approved patterns
const hasApprovedPattern = APPROVED_MOCK_PATTERNS.some(pattern => pattern.test(line));
if (hasApprovedPattern) {
return false;
}
// Check for unapproved patterns
return UNAPPROVED_MOCK_PATTERNS.some(pattern => pattern.test(line));
}
// Combined pattern checker for backward compatibility
const VITEST_MOCKING_PATTERNS = VITEST_GENERIC_PATTERNS;
// CRITICAL: ARCHITECTURAL VIOLATIONS - Utilities bypassing CommandExecutor (BANNED)
const UTILITY_BYPASS_PATTERNS = [
/spawn\s*\(/, // Direct Node.js spawn usage in utilities - BANNED
/exec\s*\(/, // Direct Node.js exec usage in utilities - BANNED
/execSync\s*\(/, // Direct Node.js execSync usage in utilities - BANNED
/child_process\./, // Direct child_process module usage in utilities - BANNED
];
// TypeScript patterns are now handled by ESLint - removed from domain-specific checks
// ESLint has comprehensive TypeScript rules with proper test file exceptions
// CRITICAL: HANDLER SIGNATURE VIOLATIONS ARE FORBIDDEN
// MCP SDK requires handlers to have exact signatures:
// Tools: (args: Record<string, unknown>) => Promise<ToolResponse>
// Resources: (uri: URL) => Promise<{ contents: Array<{ text: string }> }>
const HANDLER_SIGNATURE_VIOLATIONS = [
/async\s+handler\s*\([^)]*:\s*[^,)]+,\s*[^)]+\s*:/ms, // Handler with multiple parameters separated by comma - BANNED
/async\s+handler\s*\(\s*args\?\s*:/ms, // Handler with optional args parameter - BANNED (should be required)
/async\s+handler\s*\([^)]*,\s*[^)]*CommandExecutor/ms, // Handler with CommandExecutor parameter - BANNED
/async\s+handler\s*\([^)]*,\s*[^)]*FileSystemExecutor/ms, // Handler with FileSystemExecutor parameter - BANNED
/async\s+handler\s*\([^)]*,\s*[^)]*Dependencies/ms, // Handler with Dependencies parameter - BANNED
/async\s+handler\s*\([^)]*,\s*[^)]*executor\s*:/ms, // Handler with executor parameter - BANNED
/async\s+handler\s*\([^)]*,\s*[^)]*dependencies\s*:/ms, // Handler with dependencies parameter - BANNED
];
// CRITICAL: HANDLER TESTING IN TESTS IS FORBIDDEN
// Tests must ONLY call logic functions with dependency injection, NEVER handlers directly
// Handlers are thin wrappers for MCP SDK - testing them violates dependency injection architecture
const HANDLER_TESTING_VIOLATIONS = [
/\.handler\s*\(/, // Direct handler calls in tests - BANNED
/await\s+\w+\.handler\s*\(/, // Awaited handler calls - BANNED
/const\s+result\s*=\s*await\s+\w+\.handler/, // Handler result assignment - BANNED
/expect\s*\(\s*await\s+\w+\.handler/, // Handler expectation calls - BANNED
];
// CRITICAL: IMPROPER SERVER TYPING PATTERNS ARE FORBIDDEN
// Server instances must use proper MCP SDK types, not generic Record<string, unknown> casts
const IMPROPER_SERVER_TYPING_VIOLATIONS = [
/as Record<string, unknown>.*server/, // Casting server to Record - BANNED
/server.*as Record<string, unknown>/, // Casting server to Record - BANNED
/mcpServer\?\s*:\s*Record<string, unknown>/, // Typing server as Record - BANNED
/server\.server\?\?\s*server.*as Record/, // Complex server casting - BANNED
/interface\s+MCPServerInterface\s*{/, // Custom MCP interfaces when SDK types exist - BANNED
];
// ALLOWED PATTERNS for cleanup (not mocking)
const ALLOWED_CLEANUP_PATTERNS = [
// All cleanup patterns removed - no exceptions allowed
];
// Patterns that indicate TRUE dependency injection approach
const DEPENDENCY_INJECTION_PATTERNS = [
/createMockExecutor/, // createMockExecutor usage
/createMockFileSystemExecutor/, // createMockFileSystemExecutor usage
/executor\?\s*:\s*CommandExecutor/, // executor?: CommandExecutor parameter
];
function findTestFiles(dir) {
const testFiles = [];
function traverse(currentDir) {
const items = readdirSync(currentDir);
for (const item of items) {
const fullPath = join(currentDir, item);
const stat = statSync(fullPath);
if (stat.isDirectory()) {
// Skip node_modules and other non-relevant directories
if (!item.startsWith('.') && item !== 'node_modules' && item !== 'dist' && item !== 'build') {
traverse(fullPath);
}
} else if (item.endsWith('.test.ts') || item.endsWith('.test.js')) {
testFiles.push(fullPath);
}
}
}
traverse(dir);
return testFiles;
}
function findToolAndResourceFiles(dir) {
const toolFiles = [];
function traverse(currentDir) {
const items = readdirSync(currentDir);
for (const item of items) {
const fullPath = join(currentDir, item);
const stat = statSync(fullPath);
if (stat.isDirectory()) {
// Skip test directories and other non-relevant directories
if (!item.startsWith('.') && item !== '__tests__' && item !== 'node_modules' && item !== 'dist' && item !== 'build') {
traverse(fullPath);
}
} else if ((item.endsWith('.ts') || item.endsWith('.js')) && !item.includes('.test.') && item !== 'index.ts' && item !== 'index.js') {
toolFiles.push(fullPath);
}
}
}
traverse(dir);
return toolFiles;
}
function findUtilityFiles(dir) {
const utilityFiles = [];
function traverse(currentDir) {
const items = readdirSync(currentDir);
for (const item of items) {
const fullPath = join(currentDir, item);
const stat = statSync(fullPath);
if (stat.isDirectory()) {
// Skip test directories and other non-relevant directories
if (!item.startsWith('.') && item !== '__tests__' && item !== 'node_modules' && item !== 'dist' && item !== 'build') {
traverse(fullPath);
}
} else if ((item.endsWith('.ts') || item.endsWith('.js')) && !item.includes('.test.') && item !== 'index.ts' && item !== 'index.js') {
// Only include key utility files that should use CommandExecutor
// Exclude command.ts itself as it's the core implementation that is allowed to use spawn()
if (fullPath.includes('/utils/') && (
fullPath.includes('log_capture.ts') ||
fullPath.includes('build.ts') ||
fullPath.includes('simctl.ts')
) && !fullPath.includes('command.ts')) {
utilityFiles.push(fullPath);
}
}
}
}
traverse(dir);
return utilityFiles;
}
// Helper function to determine if a file is a test file
function isTestFile(filePath) {
return filePath.includes('__tests__') || filePath.endsWith('.test.ts') || filePath.endsWith('.test.js');
}
// Helper function to determine if a file is a production file
function isProductionFile(filePath) {
return !isTestFile(filePath) && (filePath.endsWith('.ts') || filePath.endsWith('.js'));
}
// Helper function to determine if a file is allowed to use child_process
function isAllowedChildProcessFile(filePath) {
// These files need direct child_process access for their core functionality
return filePath.includes('command.ts') || // Core CommandExecutor implementation
filePath.includes('swift_package_run.ts'); // Needs spawn for background process management
}
function analyzeTestFile(filePath) {
try {
const content = readFileSync(filePath, 'utf8');
const relativePath = relative(projectRoot, filePath);
// Check for vitest mocking patterns using new robust approach
const vitestMockingDetails = [];
const lines = content.split('\n');
// 1. Check generic vi.* patterns (always violations)
lines.forEach((line, index) => {
VITEST_GENERIC_PATTERNS.forEach(pattern => {
if (pattern.test(line)) {
vitestMockingDetails.push({
line: index + 1,
content: line.trim(),
pattern: pattern.source,
type: 'vitest-generic'
});
}
});
// 2. Check for unapproved mock patterns
if (hasUnapprovedMockPattern(line)) {
// Find which specific pattern matched for better reporting
const matchedPattern = UNAPPROVED_MOCK_PATTERNS.find(pattern => pattern.test(line));
vitestMockingDetails.push({
line: index + 1,
content: line.trim(),
pattern: matchedPattern ? matchedPattern.source : 'unapproved mock pattern',
type: 'unapproved-mock'
});
}
});
const hasVitestMockingPatterns = vitestMockingDetails.length > 0;
// TypeScript patterns now handled by ESLint
const hasTypescriptAntipatterns = false;
// Check for handler testing violations (FORBIDDEN - ARCHITECTURAL VIOLATION)
const hasHandlerTestingViolations = HANDLER_TESTING_VIOLATIONS.some(pattern => pattern.test(content));
// Check for improper server typing violations (FORBIDDEN - ARCHITECTURAL VIOLATION)
const hasImproperServerTypingViolations = IMPROPER_SERVER_TYPING_VIOLATIONS.some(pattern => pattern.test(content));
// Check for dependency injection patterns (TRUE DI)
const hasDIPatterns = DEPENDENCY_INJECTION_PATTERNS.some(pattern => pattern.test(content));
// Extract specific pattern occurrences for details
const execSyncDetails = []; // Not applicable to test files
const typescriptAntipatternDetails = []; // Unused - TypeScript handled by ESLint
const handlerTestingDetails = [];
const improperServerTypingDetails = [];
lines.forEach((line, index) => {
// TypeScript anti-patterns now handled by ESLint - removed from domain checks
HANDLER_TESTING_VIOLATIONS.forEach(pattern => {
if (pattern.test(line)) {
handlerTestingDetails.push({
line: index + 1,
content: line.trim(),
pattern: pattern.source
});
}
});
IMPROPER_SERVER_TYPING_VIOLATIONS.forEach(pattern => {
if (pattern.test(line)) {
improperServerTypingDetails.push({
line: index + 1,
content: line.trim(),
pattern: pattern.source
});
}
});
});
return {
filePath: relativePath,
hasExecSyncPatterns: false, // Not applicable to test files
hasVitestMockingPatterns,
hasTypescriptAntipatterns,
hasHandlerTestingViolations,
hasImproperServerTypingViolations,
hasDIPatterns,
execSyncDetails,
vitestMockingDetails,
typescriptAntipatternDetails,
handlerTestingDetails,
improperServerTypingDetails,
needsConversion: hasVitestMockingPatterns || hasHandlerTestingViolations || hasImproperServerTypingViolations,
isConverted: hasDIPatterns && !hasVitestMockingPatterns && !hasHandlerTestingViolations && !hasImproperServerTypingViolations,
isMixed: (hasVitestMockingPatterns || hasHandlerTestingViolations || hasImproperServerTypingViolations) && hasDIPatterns
};
} catch (error) {
console.error(`Error reading file ${filePath}: ${error.message}`);
return null;
}
}
function analyzeToolOrResourceFile(filePath) {
try {
const content = readFileSync(filePath, 'utf8');
const relativePath = relative(projectRoot, filePath);
// Check for execSync patterns in production code (excluding allowed files)
const hasExecSyncPatterns = isProductionFile(filePath) &&
!isAllowedChildProcessFile(filePath) &&
EXECSYNC_PATTERNS.some(pattern => pattern.test(content));
// Check for vitest mocking patterns using new robust approach
const vitestMockingDetails = [];
const lines = content.split('\n');
// 1. Check generic vi.* patterns (always violations)
lines.forEach((line, index) => {
VITEST_GENERIC_PATTERNS.forEach(pattern => {
if (pattern.test(line)) {
vitestMockingDetails.push({
line: index + 1,
content: line.trim(),
pattern: pattern.source,
type: 'vitest-generic'
});
}
});
// 2. Check for unapproved mock patterns
if (hasUnapprovedMockPattern(line)) {
// Find which specific pattern matched for better reporting
const matchedPattern = UNAPPROVED_MOCK_PATTERNS.find(pattern => pattern.test(line));
vitestMockingDetails.push({
line: index + 1,
content: line.trim(),
pattern: matchedPattern ? matchedPattern.source : 'unapproved mock pattern',
type: 'unapproved-mock'
});
}
});
const hasVitestMockingPatterns = vitestMockingDetails.length > 0;
// TypeScript patterns now handled by ESLint
const hasTypescriptAntipatterns = false;
// Check for dependency injection patterns (TRUE DI)
const hasDIPatterns = DEPENDENCY_INJECTION_PATTERNS.some(pattern => pattern.test(content));
// Check for handler signature violations (FORBIDDEN)
const hasHandlerSignatureViolations = HANDLER_SIGNATURE_VIOLATIONS.some(pattern => pattern.test(content));
// Check for improper server typing violations (FORBIDDEN - ARCHITECTURAL VIOLATION)
const hasImproperServerTypingViolations = IMPROPER_SERVER_TYPING_VIOLATIONS.some(pattern => pattern.test(content));
// Check for utility bypass patterns (ARCHITECTURAL VIOLATION)
const hasUtilityBypassPatterns = UTILITY_BYPASS_PATTERNS.some(pattern => pattern.test(content));
// Extract specific pattern occurrences for details
const execSyncDetails = [];
const typescriptAntipatternDetails = []; // Unused - TypeScript handled by ESLint
const handlerSignatureDetails = [];
const improperServerTypingDetails = [];
const utilityBypassDetails = [];
lines.forEach((line, index) => {
if (isProductionFile(filePath) && !isAllowedChildProcessFile(filePath)) {
EXECSYNC_PATTERNS.forEach(pattern => {
if (pattern.test(line)) {
execSyncDetails.push({
line: index + 1,
content: line.trim(),
pattern: pattern.source
});
}
});
}
// TypeScript anti-patterns now handled by ESLint - removed from domain checks
IMPROPER_SERVER_TYPING_VIOLATIONS.forEach(pattern => {
if (pattern.test(line)) {
improperServerTypingDetails.push({
line: index + 1,
content: line.trim(),
pattern: pattern.source
});
}
});
UTILITY_BYPASS_PATTERNS.forEach(pattern => {
if (pattern.test(line)) {
utilityBypassDetails.push({
line: index + 1,
content: line.trim(),
pattern: pattern.source
});
}
});
});
if (hasHandlerSignatureViolations) {
// Use regex to find the violation and its line number
const lines = content.split('\n');
const fullContent = content;
HANDLER_SIGNATURE_VIOLATIONS.forEach(pattern => {
let match;
const globalPattern = new RegExp(pattern.source, pattern.flags + 'g');
while ((match = globalPattern.exec(fullContent)) !== null) {
// Find which line this match is on
const beforeMatch = fullContent.substring(0, match.index);
const lineNumber = beforeMatch.split('\n').length;
handlerSignatureDetails.push({
line: lineNumber,
content: match[0].replace(/\s+/g, ' ').trim(),
pattern: pattern.source
});
}
});
}
return {
filePath: relativePath,
hasExecSyncPatterns,
hasVitestMockingPatterns,
hasTypescriptAntipatterns,
hasDIPatterns,
hasHandlerSignatureViolations,
hasImproperServerTypingViolations,
hasUtilityBypassPatterns,
execSyncDetails,
vitestMockingDetails,
typescriptAntipatternDetails,
handlerSignatureDetails,
improperServerTypingDetails,
utilityBypassDetails,
needsConversion: hasExecSyncPatterns || hasVitestMockingPatterns || hasHandlerSignatureViolations || hasImproperServerTypingViolations || hasUtilityBypassPatterns,
isConverted: hasDIPatterns && !hasExecSyncPatterns && !hasVitestMockingPatterns && !hasHandlerSignatureViolations && !hasImproperServerTypingViolations && !hasUtilityBypassPatterns,
isMixed: (hasExecSyncPatterns || hasVitestMockingPatterns || hasHandlerSignatureViolations || hasImproperServerTypingViolations || hasUtilityBypassPatterns) && hasDIPatterns
};
} catch (error) {
console.error(`Error reading file ${filePath}: ${error.message}`);
return null;
}
}
function main() {
console.log('🔍 XcodeBuildMCP Code Pattern Violations Checker\n');
console.log(`🎯 Checking pattern type: ${patternFilter.toUpperCase()}\n`);
console.log('CODE GUIDELINES ENFORCED:');
console.log('✅ ONLY ALLOWED: createMockExecutor() and createMockFileSystemExecutor()');
console.log('❌ BANNED: vitest mocking patterns (vi.mock, vi.fn, .mockResolvedValue, etc.)');
console.log('❌ BANNED: execSync usage in production code (use CommandExecutor instead)');
console.log('ℹ️ TypeScript patterns: Handled by ESLint with proper test exceptions');
console.log('❌ BANNED: handler signature violations (handlers must have exact MCP SDK signatures)');
console.log('❌ BANNED: handler testing violations (test logic functions, not handlers directly)');
console.log('❌ BANNED: improper server typing (use McpServer type, not Record<string, unknown>)\n');
const testFiles = findTestFiles(join(projectRoot, 'src'));
const testResults = testFiles.map(analyzeTestFile).filter(Boolean);
// Also check tool and resource files for TypeScript anti-patterns AND handler signature violations
const toolFiles = findToolAndResourceFiles(join(projectRoot, 'src', 'mcp', 'tools'));
const resourceFiles = findToolAndResourceFiles(join(projectRoot, 'src', 'mcp', 'resources'));
const allToolAndResourceFiles = [...toolFiles, ...resourceFiles];
const toolResults = allToolAndResourceFiles.map(analyzeToolOrResourceFile).filter(Boolean);
// Check utility files for architectural violations (bypassing CommandExecutor)
const utilityFiles = findUtilityFiles(join(projectRoot, 'src'));
const utilityResults = utilityFiles.map(analyzeToolOrResourceFile).filter(Boolean);
// Combine test, tool, and utility file results for analysis
const results = [...testResults, ...toolResults, ...utilityResults];
const handlerResults = toolResults;
const utilityBypassResults = utilityResults.filter(r => r.hasUtilityBypassPatterns);
// Filter results based on pattern type
let filteredResults;
let filteredHandlerResults = [];
switch (patternFilter) {
case 'vitest':
filteredResults = results.filter(r => r.hasVitestMockingPatterns);
console.log(`Filtering to show only vitest mocking violations (${filteredResults.length} files)`);
break;
case 'execsync':
filteredResults = results.filter(r => r.hasExecSyncPatterns);
console.log(`Filtering to show only execSync violations (${filteredResults.length} files)`);
break;
// TypeScript case removed - now handled by ESLint
case 'handler':
filteredResults = [];
filteredHandlerResults = handlerResults.filter(r => r.hasHandlerSignatureViolations);
console.log(`Filtering to show only handler signature violations (${filteredHandlerResults.length} files)`);
break;
case 'handler-testing':
filteredResults = results.filter(r => r.hasHandlerTestingViolations);
console.log(`Filtering to show only handler testing violations (${filteredResults.length} files)`);
break;
case 'server-typing':
filteredResults = results.filter(r => r.hasImproperServerTypingViolations);
console.log(`Filtering to show only improper server typing violations (${filteredResults.length} files)`);
break;
case 'all':
default:
filteredResults = results.filter(r => r.needsConversion);
filteredHandlerResults = handlerResults.filter(r => r.hasHandlerSignatureViolations);
console.log(`Showing all pattern violations (${filteredResults.length} test files + ${filteredHandlerResults.length} handler files)`);
break;
}
const needsConversion = filteredResults;
const converted = results.filter(r => r.isConverted);
const mixed = results.filter(r => r.isMixed);
const execSyncOnly = results.filter(r => r.hasExecSyncPatterns && !r.hasVitestMockingPatterns && true && !r.hasHandlerTestingViolations && !r.hasImproperServerTypingViolations && !r.hasDIPatterns);
const vitestMockingOnly = results.filter(r => r.hasVitestMockingPatterns && !r.hasExecSyncPatterns && true && !r.hasHandlerTestingViolations && !r.hasImproperServerTypingViolations && !r.hasDIPatterns);
const typescriptOnly = results.filter(r => r.false && !r.hasExecSyncPatterns && !r.hasVitestMockingPatterns && !r.hasHandlerTestingViolations && !r.hasImproperServerTypingViolations && !r.hasDIPatterns);
const handlerTestingOnly = results.filter(r => r.hasHandlerTestingViolations && !r.hasExecSyncPatterns && !r.hasVitestMockingPatterns && true && !r.hasImproperServerTypingViolations && !r.hasDIPatterns);
const improperServerTypingOnly = results.filter(r => r.hasImproperServerTypingViolations && !r.hasExecSyncPatterns && !r.hasVitestMockingPatterns && !r.hasHandlerTestingViolations && !r.hasDIPatterns);
const noPatterns = results.filter(r => !r.hasExecSyncPatterns && !r.hasVitestMockingPatterns && true && !r.hasHandlerTestingViolations && !r.hasImproperServerTypingViolations && !r.hasDIPatterns);
console.log(`📊 CODE PATTERN VIOLATION ANALYSIS`);
console.log(`=================================`);
console.log(`Total files analyzed: ${results.length}`);
console.log(`🚨 FILES WITH VIOLATIONS: ${needsConversion.length}`);
console.log(` └─ execSync production violations: ${execSyncOnly.length}`);
console.log(` └─ vitest mocking violations: ${vitestMockingOnly.length}`);
// TypeScript anti-patterns now handled by ESLint
console.log(` └─ handler testing violations: ${handlerTestingOnly.length}`);
console.log(` └─ improper server typing violations: ${improperServerTypingOnly.length}`);
console.log(`🚨 ARCHITECTURAL VIOLATIONS: ${utilityBypassResults.length}`);
console.log(`✅ COMPLIANT (best practices): ${converted.length}`);
console.log(`⚠️ MIXED VIOLATIONS: ${mixed.length}`);
console.log(`📝 No patterns detected: ${noPatterns.length}`);
console.log('');
if (needsConversion.length > 0) {
console.log(`❌ FILES THAT NEED CONVERSION (${needsConversion.length}):`);
console.log(`=====================================`);
needsConversion.forEach((result, index) => {
console.log(`${index + 1}. ${result.filePath}`);
if (result.execSyncDetails && result.execSyncDetails.length > 0) {
console.log(` 🚨 EXECSYNC PATTERNS (${result.execSyncDetails.length}):`);
result.execSyncDetails.slice(0, 2).forEach(detail => {
console.log(` Line ${detail.line}: ${detail.content}`);
});
if (result.execSyncDetails.length > 2) {
console.log(` ... and ${result.execSyncDetails.length - 2} more execSync patterns`);
}
console.log(` 🔧 FIX: Replace execSync with CommandExecutor dependency injection`);
}
if (result.vitestMockingDetails.length > 0) {
console.log(` 🧪 VITEST MOCKING PATTERNS (${result.vitestMockingDetails.length}):`);
result.vitestMockingDetails.slice(0, 2).forEach(detail => {
console.log(` Line ${detail.line}: ${detail.content}`);
});
if (result.vitestMockingDetails.length > 2) {
console.log(` ... and ${result.vitestMockingDetails.length - 2} more vitest patterns`);
}
}
// TypeScript violations now handled by ESLint - removed from domain checks
if (result.handlerTestingDetails && result.handlerTestingDetails.length > 0) {
console.log(` 🚨 HANDLER TESTING VIOLATIONS (${result.handlerTestingDetails.length}):`);
result.handlerTestingDetails.slice(0, 2).forEach(detail => {
console.log(` Line ${detail.line}: ${detail.content}`);
});
if (result.handlerTestingDetails.length > 2) {
console.log(` ... and ${result.handlerTestingDetails.length - 2} more handler testing violations`);
}
console.log(` 🔧 FIX: Replace handler calls with logic function calls using dependency injection`);
}
if (result.improperServerTypingDetails && result.improperServerTypingDetails.length > 0) {
console.log(` 🔧 IMPROPER SERVER TYPING VIOLATIONS (${result.improperServerTypingDetails.length}):`);
result.improperServerTypingDetails.slice(0, 2).forEach(detail => {
console.log(` Line ${detail.line}: ${detail.content}`);
});
if (result.improperServerTypingDetails.length > 2) {
console.log(` ... and ${result.improperServerTypingDetails.length - 2} more server typing violations`);
}
console.log(` 🔧 FIX: Import McpServer from SDK and use proper typing instead of Record<string, unknown>`);
}
console.log('');
});
}
// Utility bypass violations reporting
if (utilityBypassResults.length > 0) {
console.log(`🚨 CRITICAL: UTILITY ARCHITECTURAL VIOLATIONS (${utilityBypassResults.length}):`);
console.log(`=======================================================`);
console.log('⚠️ These utilities bypass CommandExecutor and break our testing architecture!');
console.log('');
utilityBypassResults.forEach((result, index) => {
console.log(`${index + 1}. ${result.filePath}`);
if (result.utilityBypassDetails.length > 0) {
console.log(` 🚨 BYPASS PATTERNS (${result.utilityBypassDetails.length}):`);
result.utilityBypassDetails.forEach(detail => {
console.log(` Line ${detail.line}: ${detail.content}`);
});
}
console.log(' 🔧 FIX: Refactor to accept CommandExecutor and use it instead of direct spawn/exec calls');
console.log('');
});
}
// Handler signature violations reporting
if (filteredHandlerResults.length > 0) {
console.log(`🚨 HANDLER SIGNATURE VIOLATIONS (${filteredHandlerResults.length}):`);
console.log(`===========================================`);
filteredHandlerResults.forEach((result, index) => {
console.log(`${index + 1}. ${result.filePath}`);
if (result.handlerSignatureDetails.length > 0) {
console.log(` 🛠️ HANDLER VIOLATIONS (${result.handlerSignatureDetails.length}):`);
result.handlerSignatureDetails.forEach(detail => {
console.log(` Line ${detail.line}: ${detail.content}`);
});
}
console.log('');
});
}
if (mixed.length > 0) {
console.log(`⚠️ FILES WITH MIXED PATTERNS (${mixed.length}):`);
console.log(`===================================`);
mixed.forEach((result, index) => {
console.log(`${index + 1}. ${result.filePath}`);
console.log(` ⚠️ Contains both setTimeout and dependency injection patterns`);
console.log('');
});
}
if (converted.length > 0) {
console.log(`✅ SUCCESSFULLY CONVERTED FILES (${converted.length}):`);
console.log(`====================================`);
converted.forEach((result, index) => {
console.log(`${index + 1}. ${result.filePath}`);
});
console.log('');
}
// Summary for next steps
const hasViolations = needsConversion.length > 0 || filteredHandlerResults.length > 0 || utilityBypassResults.length > 0;
if (needsConversion.length > 0) {
console.log(`🚨 CRITICAL ACTION REQUIRED (TEST FILES):`);
console.log(`=======================================`);
console.log(`1. IMMEDIATELY remove ALL vitest mocking from ${needsConversion.length} files`);
console.log(`2. BANNED: vi.mock(), vi.fn(), .mockResolvedValue(), .toHaveBeenCalled(), etc.`);
console.log(`3. BANNED: Testing handlers directly (.handler()) - test logic functions with dependency injection`);
console.log(`4. ONLY ALLOWED: createMockExecutor() and createMockFileSystemExecutor()`);
console.log(`4. Update plugin implementations to accept executor?: CommandExecutor parameter`);
console.log(`5. Run this script again after each fix to track progress`);
console.log('');
// Show top files by total violation count
const sortedByPatterns = needsConversion
.sort((a, b) => {
const totalA = (a.execSyncDetails?.length || 0) + a.vitestMockingDetails.length + (a.handlerTestingDetails?.length || 0) + (a.improperServerTypingDetails?.length || 0);
const totalB = (b.execSyncDetails?.length || 0) + b.vitestMockingDetails.length + (b.handlerTestingDetails?.length || 0) + (b.improperServerTypingDetails?.length || 0);
return totalB - totalA;
})
.slice(0, 5);
console.log(`🚨 TOP 5 FILES WITH MOST VIOLATIONS:`);
sortedByPatterns.forEach((result, index) => {
const totalPatterns = (result.execSyncDetails?.length || 0) + result.vitestMockingDetails.length + (result.handlerTestingDetails?.length || 0) + (result.improperServerTypingDetails?.length || 0);
console.log(`${index + 1}. ${result.filePath} (${totalPatterns} violations: ${result.execSyncDetails?.length || 0} execSync + ${result.vitestMockingDetails.length} vitest + ${result.handlerTestingDetails?.length || 0} handler + ${result.improperServerTypingDetails?.length || 0} server)`);
});
console.log('');
}
if (utilityBypassResults.length > 0) {
console.log(`🚨 CRITICAL ACTION REQUIRED (UTILITY FILES):`);
console.log(`==========================================`);
console.log(`1. IMMEDIATELY fix ALL architectural violations in ${utilityBypassResults.length} files`);
console.log(`2. Refactor utilities to accept CommandExecutor parameter`);
console.log(`3. Replace direct spawn/exec calls with executor calls`);
console.log(`4. These violations break our entire testing strategy`);
console.log(`5. Run this script again after each fix to track progress`);
console.log('');
}
if (filteredHandlerResults.length > 0) {
console.log(`🚨 CRITICAL ACTION REQUIRED (HANDLER FILES):`);
console.log(`==========================================`);
console.log(`1. IMMEDIATELY fix ALL handler signature violations in ${filteredHandlerResults.length} files`);
console.log(`2. Tools: Handler must be: async handler(args: Record<string, unknown>): Promise<ToolResponse>`);
console.log(`3. Resources: Handler must be: async handler(uri: URL): Promise<{ contents: Array<{ text: string }> }>`);
console.log(`4. Inject dependencies INSIDE handler body: const executor = getDefaultCommandExecutor()`);
console.log(`5. Run this script again after each fix to track progress`);
console.log('');
}
if (!hasViolations && mixed.length === 0) {
console.log(`🎉 ALL FILES COMPLY WITH PROJECT STANDARDS!`);
console.log(`==========================================`);
console.log(`✅ All files use ONLY createMockExecutor() and createMockFileSystemExecutor()`);
console.log(`✅ All files follow TypeScript best practices (no unsafe casts)`);
console.log(`✅ All handler signatures comply with MCP SDK requirements`);
console.log(`✅ All utilities properly use CommandExecutor dependency injection`);
console.log(`✅ No violations detected!`);
}
// Exit with appropriate code
process.exit(hasViolations || mixed.length > 0 ? 1 : 0);
}
main();
```
--------------------------------------------------------------------------------
/docs/dev/TESTING.md:
--------------------------------------------------------------------------------
```markdown
# XcodeBuildMCP Plugin Testing Guidelines
This document provides comprehensive testing guidelines for XcodeBuildMCP plugins, ensuring consistent, robust, and maintainable test coverage across the entire codebase.
## Table of Contents
1. [Testing Philosophy](#testing-philosophy)
2. [Test Architecture](#test-architecture)
3. [Dependency Injection Strategy](#dependency-injection-strategy)
4. [Three-Dimensional Testing](#three-dimensional-testing)
5. [Test Organization](#test-organization)
6. [Test Patterns](#test-patterns)
7. [Performance Requirements](#performance-requirements)
8. [Coverage Standards](#coverage-standards)
9. [Common Patterns](#common-patterns)
10. [Manual Testing with Reloaderoo](#manual-testing-with-reloaderoo)
11. [Troubleshooting](#troubleshooting)
## Testing Philosophy
### 🚨 CRITICAL: No Vitest Mocking Allowed
### ABSOLUTE RULE: ALL VITEST MOCKING IS COMPLETELY BANNED
### FORBIDDEN PATTERNS (will cause immediate test failure):
#### Vitest Mocking (COMPLETELY BANNED):
- `vi.mock()` - BANNED
- `vi.fn()` - BANNED
- `vi.mocked()` - BANNED
- `vi.spyOn()` - BANNED
- `.mockResolvedValue()` - BANNED
- `.mockRejectedValue()` - BANNED
- `.mockReturnValue()` - BANNED
- `.mockImplementation()` - BANNED
- `.toHaveBeenCalled()` - BANNED
- `.toHaveBeenCalledWith()` - BANNED
- `MockedFunction` type - BANNED
#### Manual Mock Implementations (BANNED - use our utilities instead):
- `const mockExecutor = async (...) => { ... }` - Use `createMockExecutor()` instead
- `const mockFsDeps = { readFile: async () => ... }` - Use `createMockFileSystemExecutor()` instead
- `const mockServer = { ... }` - Refactor to use dependency injection pattern
- Any manual async function implementations for mocking behavior
### ONLY ALLOWED MOCKING:
- `createMockExecutor({ success: true, output: 'result' })` - command execution
- `createMockFileSystemExecutor({ readFile: async () => 'content' })` - file system operations
### OUR CORE PRINCIPLE
**Simple Rule**: No mocking other than `createMockExecutor()` and `createMockFileSystemExecutor()` (and their noop variants).
**Why This Rule Exists**:
1. **Consistency**: All tests use the same mocking utilities, making them predictable and maintainable
2. **Reliability**: Our utilities are thoroughly tested and handle edge cases properly
3. **Architectural Enforcement**: Prevents bypassing our dependency injection patterns
4. **Simplicity**: One clear rule instead of complex guidelines about what mocking is acceptable
### Integration Testing with Dependency Injection
XcodeBuildMCP follows a **pure dependency injection** testing philosophy that eliminates vitest mocking:
- ✅ **Test plugin interfaces** (public API contracts)
- ✅ **Test integration flows** (plugin → utilities → external tools)
- ✅ **Use dependency injection** with createMockExecutor()
- ❌ **Never mock vitest functions** (vi.mock, vi.fn, etc.)
### Benefits
1. **Implementation Independence**: Internal refactoring doesn't break tests
2. **Real Coverage**: Tests verify actual user data flows
3. **Maintainability**: No brittle vitest mocks that break on implementation changes
4. **True Integration**: Catches integration bugs between layers
5. **Test Safety**: Default executors throw errors in test environment
### Automated Violation Checking
To enforce the no-mocking policy, the project includes a script that automatically checks for banned testing patterns.
```bash
# Run the script to check for violations
node scripts/check-code-patterns.js
```
This script is part of the standard development workflow and should be run before committing changes to ensure compliance with the testing standards.
### What the Script Flags vs. What It Should NOT Flag
#### ✅ LEGITIMATE VIOLATIONS (correctly flagged):
- Manual mock executors: `const mockExecutor = async (...) => { ... }`
- Manual filesystem mocks: `const mockFsDeps = { readFile: async () => ... }`
- Manual server mocks: `const mockServer = { ... }`
- Vitest mocking patterns: `vi.mock()`, `vi.fn()`, etc.
#### ❌ FALSE POSITIVES (should NOT be flagged):
- Test data tracking: `commandCalls.push({ ... })` - This is just collecting test data, not mocking behavior
- Regular variables: `const testData = { ... }` - Non-mocking object assignments
- Test setup: Regular const assignments that don't implement mock behavior
The script has been refined to minimize false positives while catching all legitimate violations of our core rule.
## Test Architecture
### Correct Test Flow
```
Test → Plugin Handler → utilities → [DEPENDENCY INJECTION] createMockExecutor()
```
### What Gets Tested
- Plugin parameter validation
- Business logic execution
- Command generation
- Response formatting
- Error handling
- Integration between layers
### What Gets Mocked
- Command execution via `createMockExecutor()`
- File system operations via `createMockFileSystemExecutor()`
- Nothing else - all vitest mocking is banned
## Dependency Injection Strategy
### Handler Requirements
All plugin handlers must support dependency injection:
```typescript
export function tool_nameLogic(
args: Record<string, unknown>,
commandExecutor: CommandExecutor,
fileSystemExecutor?: FileSystemExecutor
): Promise<ToolResponse> {
// Use injected executors
const result = await executeCommand(['xcrun', 'simctl', 'list'], commandExecutor);
return createTextResponse(result.output);
}
export default {
name: 'tool_name',
description: 'Tool description',
schema: { /* zod schema */ },
async handler(args: Record<string, unknown>): Promise<ToolResponse> {
return tool_nameLogic(args, getDefaultCommandExecutor(), getDefaultFileSystemExecutor());
},
};
```
**Important**: The dependency injection pattern applies to ALL handlers, including:
- Tool handlers
- Resource handlers
- Any future handler types (prompts, etc.)
Always use default parameter values (e.g., `= getDefaultCommandExecutor()`) to ensure production code works without explicit executor injection, while tests can override with mock executors.
### Test Requirements
All tests must explicitly provide mock executors:
```typescript
it('should handle successful command execution', async () => {
const mockExecutor = createMockExecutor({
success: true,
output: 'BUILD SUCCEEDED'
});
const result = await tool_nameLogic(
{ projectPath: '/test.xcodeproj', scheme: 'MyApp' },
mockExecutor
);
expect(result.content[0].text).toContain('Build succeeded');
});
```
## Three-Dimensional Testing
Every plugin test suite must validate three critical dimensions:
### 1. Input Validation (Schema Testing)
Test parameter validation and schema compliance:
```typescript
describe('Parameter Validation', () => {
it('should accept valid parameters', () => {
const schema = z.object(tool.schema);
expect(schema.safeParse({
projectPath: '/valid/path.xcodeproj',
scheme: 'ValidScheme'
}).success).toBe(true);
});
it('should reject invalid parameters', () => {
const schema = z.object(tool.schema);
expect(schema.safeParse({
projectPath: 123, // Wrong type
scheme: 'ValidScheme'
}).success).toBe(false);
});
it('should handle missing required parameters', async () => {
const mockExecutor = createMockExecutor({ success: true });
const result = await tool.handler({ scheme: 'MyApp' }, mockExecutor); // Missing projectPath
expect(result).toEqual({
content: [{
type: 'text',
text: "Required parameter 'projectPath' is missing. Please provide a value for this parameter."
}],
isError: true
});
});
});
```
### 2. Command Generation (CLI Testing)
### CRITICAL: No command spying allowed. Test command generation through response validation.
```typescript
describe('Command Generation', () => {
it('should execute correct command with minimal parameters', async () => {
const mockExecutor = createMockExecutor({
success: true,
output: 'BUILD SUCCEEDED'
});
const result = await tool.handler({
projectPath: '/test.xcodeproj',
scheme: 'MyApp'
}, mockExecutor);
// Verify through successful response - command was executed correctly
expect(result.content[0].text).toContain('Build succeeded');
});
it('should handle paths with spaces correctly', async () => {
const mockExecutor = createMockExecutor({
success: true,
output: 'BUILD SUCCEEDED'
});
const result = await tool.handler({
projectPath: '/Users/dev/My Project/app.xcodeproj',
scheme: 'MyApp'
}, mockExecutor);
// Verify successful execution (proper path handling)
expect(result.content[0].text).toContain('Build succeeded');
});
});
```
### 3. Output Processing (Response Testing)
Test response formatting and error handling:
```typescript
describe('Response Processing', () => {
it('should format successful response', async () => {
const mockExecutor = createMockExecutor({
success: true,
output: 'BUILD SUCCEEDED'
});
const result = await tool.handler({ projectPath: '/test', scheme: 'MyApp' }, mockExecutor);
expect(result).toEqual({
content: [{ type: 'text', text: '✅ Build succeeded for scheme MyApp' }]
});
});
it('should handle command failures', async () => {
const mockExecutor = createMockExecutor({
success: false,
output: 'Build failed with errors',
error: 'Compilation error'
});
const result = await tool.handler({ projectPath: '/test', scheme: 'MyApp' }, mockExecutor);
expect(result.isError).toBe(true);
expect(result.content[0].text).toContain('Build failed');
});
it('should handle executor errors', async () => {
const mockExecutor = createMockExecutor(new Error('spawn xcodebuild ENOENT'));
const result = await tool.handler({ projectPath: '/test', scheme: 'MyApp' }, mockExecutor);
expect(result).toEqual({
content: [{ type: 'text', text: 'Error during build: spawn xcodebuild ENOENT' }],
isError: true
});
});
});
```
## Test Organization
### Directory Structure
```
src/plugins/[workflow-group]/
├── __tests__/
│ ├── index.test.ts # Workflow metadata tests (canonical groups only)
│ ├── re-exports.test.ts # Re-export validation (project/workspace groups only)
│ ├── tool1.test.ts # Individual tool tests
│ ├── tool2.test.ts
│ └── ...
├── tool1.ts
├── tool2.ts
├── index.ts # Workflow metadata
└── ...
```
### Test File Types
#### 1. Tool Tests (`tool_name.test.ts`)
Test individual plugin tools with full three-dimensional coverage.
#### 2. Workflow Tests (`index.test.ts`)
Test workflow metadata for canonical groups:
```typescript
describe('simulator-workspace workflow metadata', () => {
it('should have correct workflow name', () => {
expect(workflow.name).toBe('iOS Simulator Workspace Development');
});
it('should have correct description', () => {
expect(workflow.description).toBe(
'Complete iOS development workflow for .xcworkspace files including build, test, deploy, and debug capabilities',
);
});
});
```
#### 3. Re-export Tests (`re-exports.test.ts`)
Test re-export integrity for project/workspace groups:
```typescript
describe('simulator-project re-exports', () => {
it('should re-export boot_sim from simulator-shared', () => {
expect(bootSim.name).toBe('boot_sim');
expect(typeof bootSim.handler).toBe('function');
});
});
```
## Test Patterns
### Standard Test Template
```typescript
import { vi, describe, it, expect, beforeEach } from 'vitest';
import { z } from 'zod';
// CRITICAL: NO VITEST MOCKING ALLOWED
// Import ONLY what you need - no mock setup
import tool from '../tool_name.ts';
import { createMockExecutor } from '../../utils/command.js';
describe('tool_name', () => {
describe('Export Field Validation (Literal)', () => {
it('should export correct name', () => {
expect(tool.name).toBe('tool_name');
});
it('should export correct description', () => {
expect(tool.description).toBe('Expected literal description');
});
it('should export handler function', () => {
expect(typeof tool.handler).toBe('function');
});
// Schema validation tests...
});
describe('Command Generation', () => {
it('should execute commands successfully', async () => {
const mockExecutor = createMockExecutor({
success: true,
output: 'Expected output'
});
const result = await tool.handler(validParams, mockExecutor);
expect(result.content[0].text).toContain('Expected result');
});
});
describe('Response Processing', () => {
// Output handling tests...
});
});
```
## Performance Requirements
### Test Execution Speed
- **Individual test**: < 100ms
- **Test file**: < 5 seconds
- **Full test suite**: < 20 seconds
- **No real system calls**: Tests must use mocks
### Performance Anti-Patterns
❌ **Real command execution**:
```
[INFO] Executing command: xcodebuild -showBuildSettings...
```
❌ **Long timeouts** (indicates real calls)
❌ **File system operations** (unless testing file utilities)
❌ **Network requests** (unless testing network utilities)
## Coverage Standards
### Target Coverage
- **Overall**: 95%+
- **Plugin handlers**: 100%
- **Command generation**: 100%
- **Error paths**: 100%
### Coverage Validation
```bash
# Check coverage for specific plugin group
npm run test:coverage -- plugins/simulator-workspace/
# Ensure all code paths are tested
npm run test:coverage -- --reporter=lcov
```
### Required Test Paths
Every plugin test must cover:
- ✅ **Valid parameter combinations**
- ✅ **Invalid parameter rejection**
- ✅ **Missing required parameters**
- ✅ **Successful command execution**
- ✅ **Command failure scenarios**
- ✅ **Executor error handling**
- ✅ **Output parsing edge cases**
## Common Patterns
### Testing Parameter Defaults
```typescript
it('should use default configuration when not provided', async () => {
const mockExecutor = createMockExecutor({
success: true,
output: 'BUILD SUCCEEDED'
});
const result = await tool.handler({
projectPath: '/test.xcodeproj',
scheme: 'MyApp'
// configuration intentionally omitted
}, mockExecutor);
// Verify default behavior through successful response
expect(result.content[0].text).toContain('Build succeeded');
});
```
### Testing Complex Output Parsing
```typescript
it('should extract app path from build settings', async () => {
const mockExecutor = createMockExecutor({
success: true,
output: `
CONFIGURATION_BUILD_DIR = /path/to/build
BUILT_PRODUCTS_DIR = /path/to/products
FULL_PRODUCT_NAME = MyApp.app
OTHER_SETTING = ignored_value
`
});
const result = await tool.handler({ projectPath: '/test', scheme: 'MyApp' }, mockExecutor);
expect(result.content[0].text).toContain('/path/to/products/MyApp.app');
});
```
### Testing Error Message Formatting
```typescript
it('should format validation errors correctly', async () => {
const mockExecutor = createMockExecutor({ success: true });
const result = await tool.handler({}, mockExecutor); // Missing required params
expect(result).toEqual({
content: [{
type: 'text',
text: "Required parameter 'projectPath' is missing. Please provide a value for this parameter."
}],
isError: true
});
});
```
## Manual Testing with Reloaderoo
### 🚨 CRITICAL: THOROUGHNESS OVER EFFICIENCY - NO SHORTCUTS ALLOWED
### ABSOLUTE PRINCIPLE: EVERY TOOL MUST BE TESTED INDIVIDUALLY
### 🚨 MANDATORY TESTING SCOPE - NO EXCEPTIONS
- **EVERY SINGLE TOOL** - All 83+ tools must be tested individually, one by one
- **NO REPRESENTATIVE SAMPLING** - Testing similar tools does NOT validate other tools
- **NO PATTERN RECOGNITION SHORTCUTS** - Similar-looking tools may have different behaviors
- **NO EFFICIENCY OPTIMIZATIONS** - Thoroughness is more important than speed
- **NO TIME CONSTRAINTS** - This is a long-running task with no deadline pressure
### ❌ FORBIDDEN EFFICIENCY SHORTCUTS
- **NEVER** assume testing `build_sim_id_proj` validates `build_sim_name_proj`
- **NEVER** skip tools because they "look similar" to tested ones
- **NEVER** use representative sampling instead of complete coverage
- **NEVER** stop testing due to time concerns or perceived redundancy
- **NEVER** group tools together for batch testing
- **NEVER** make assumptions about untested tools based on tested patterns
### ✅ REQUIRED COMPREHENSIVE APPROACH
1. **Individual Tool Testing**: Each tool gets its own dedicated test execution
2. **Complete Documentation**: Every tool result must be recorded, regardless of outcome
3. **Systematic Progress**: Use TodoWrite to track every single tool as tested/untested
4. **Failure Documentation**: Test tools that cannot work and mark them as failed/blocked
5. **No Assumptions**: Treat each tool as potentially unique requiring individual validation
### TESTING COMPLETENESS VALIDATION
- **Start Count**: Record exact number of tools discovered (e.g., 83 tools)
- **End Count**: Verify same number of tools have been individually tested
- **Missing Tools = Testing Failure**: If any tools remain untested, the testing is incomplete
- **TodoWrite Tracking**: Every tool must appear in todo list and be marked completed
### 🚨 CRITICAL: Black Box Testing via Reloaderoo Inspect
### DEFINITION: Black Box Testing
Black Box Testing means testing ONLY through external interfaces without any knowledge of internal implementation. For XcodeBuildMCP, this means testing exclusively through the Model Context Protocol (MCP) interface using Reloaderoo as the MCP client.
### 🚨 MANDATORY: RELOADEROO INSPECT IS THE ONLY ALLOWED TESTING METHOD
### ABSOLUTE TESTING RULES - NO EXCEPTIONS
1. **✅ ONLY ALLOWED: Reloaderoo Inspect Commands**
- `npx reloaderoo@latest inspect call-tool "TOOL_NAME" --params 'JSON' -- node build/index.js`
- `npx reloaderoo@latest inspect list-tools -- node build/index.js`
- `npx reloaderoo@latest inspect read-resource "URI" -- node build/index.js`
- `npx reloaderoo@latest inspect server-info -- node build/index.js`
- `npx reloaderoo@latest inspect ping -- node build/index.js`
2. **❌ COMPLETELY FORBIDDEN ACTIONS:**
- **NEVER** call `mcp__XcodeBuildMCP__tool_name()` functions directly
- **NEVER** use MCP server tools as if they were native functions
- **NEVER** access internal server functionality
- **NEVER** read source code to understand how tools work
- **NEVER** examine implementation files during testing
- **NEVER** diagnose internal server issues or registration problems
- **NEVER** suggest code fixes or implementation changes
3. **🚨 CRITICAL VIOLATION EXAMPLES:**
```typescript
// ❌ FORBIDDEN - Direct MCP tool calls
await mcp__XcodeBuildMCP__list_devices();
await mcp__XcodeBuildMCP__build_sim_id_proj({ ... });
// ❌ FORBIDDEN - Using tools as native functions
const devices = await list_devices();
const result = await doctor();
// ✅ CORRECT - Only through Reloaderoo inspect
npx reloaderoo@latest inspect call-tool "list_devices" --params '{}' -- node build/index.js
npx reloaderoo@latest inspect call-tool "doctor" --params '{}' -- node build/index.js
```
### WHY RELOADEROO INSPECT IS MANDATORY
- **Higher Fidelity**: Provides clear input/output visibility for each tool call
- **Real-world Simulation**: Tests exactly how MCP clients interact with the server
- **Interface Validation**: Ensures MCP protocol compliance and proper JSON formatting
- **Black Box Enforcement**: Prevents accidental access to internal implementation details
- **Clean State**: Each tool call runs with a fresh MCP server instance, preventing cross-contamination
### IMPORTANT: STATEFUL TOOL LIMITATIONS
#### Reloaderoo Inspect Behavior:
Reloaderoo starts a fresh MCP server instance for each individual tool call and terminates it immediately after the response. This ensures:
- ✅ **Clean Testing Environment**: No state contamination between tool calls
- ✅ **Isolated Testing**: Each tool test is independent and repeatable
- ✅ **Real-world Accuracy**: Simulates how most MCP clients interact with servers
#### Expected False Negatives:
Some tools rely on in-memory state within the MCP server and will fail when tested via Reloaderoo inspect. These failures are **expected and acceptable** as false negatives:
- **`swift_package_stop`** - Requires in-memory process tracking from `swift_package_run`
- **`stop_app_device`** - Requires in-memory process tracking from `launch_app_device`
- **`stop_app_sim`** - Requires in-memory process tracking from `launch_app_sim`
- **`stop_device_log_cap`** - Requires in-memory session tracking from `start_device_log_cap`
- **`stop_sim_log_cap`** - Requires in-memory session tracking from `start_sim_log_cap`
- **`stop_mac_app`** - Requires in-memory process tracking from `launch_mac_app`
#### Testing Protocol for Stateful Tools:
1. **Test the tool anyway** - Execute the Reloaderoo inspect command
2. **Expect failure** - Tool will likely fail due to missing state
3. **Mark as false negative** - Document the failure as expected due to stateful limitations
4. **Continue testing** - Do not attempt to fix or investigate the failure
5. **Report as finding** - Note in testing report that stateful tools failed as expected
### COMPLETE COVERAGE REQUIREMENTS
- ✅ **Test ALL 83+ tools individually** - No exceptions, every tool gets manual verification
- ✅ **Follow dependency graphs** - Test tools in correct order based on data dependencies
- ✅ **Capture key outputs** - Record UUIDs, paths, schemes needed by dependent tools
- ✅ **Test real workflows** - Complete end-to-end workflows from discovery to execution
- ✅ **Use programmatic JSON parsing** - Accurate tool/resource counting and discovery
- ✅ **Document all observations** - Record exactly what you see via testing
- ✅ **Report discrepancies as findings** - Note unexpected results without investigation
### MANDATORY INDIVIDUAL TOOL TESTING PROTOCOL
#### Step 1: Create Complete Tool Inventory
```bash
# Generate complete list of all tools
npx reloaderoo@latest inspect list-tools -- node build/index.js > /tmp/all_tools.json
TOTAL_TOOLS=$(jq '.tools | length' /tmp/all_tools.json)
echo "TOTAL TOOLS TO TEST: $TOTAL_TOOLS"
# Extract all tool names for systematic testing
jq -r '.tools[].name' /tmp/all_tools.json > /tmp/tool_names.txt
```
#### Step 2: Create TodoWrite Task List for Every Tool
```bash
# Create individual todo items for each of the 83+ tools
# Example for first few tools:
# 1. [ ] Test tool: doctor
# 2. [ ] Test tool: list_devices
# 3. [ ] Test tool: list_sims
# ... (continue for ALL 83+ tools)
```
#### Step 3: Test Each Tool Individually
For EVERY tool in the list:
```bash
# Test each tool individually - NO BATCHING
npx reloaderoo@latest inspect call-tool "TOOL_NAME" --params 'APPROPRIATE_PARAMS' -- node build/index.js
# Mark tool as completed in TodoWrite IMMEDIATELY after testing
# Record result (success/failure/blocked) for each tool
```
#### Step 4: Validate Complete Coverage
```bash
# Verify all tools tested
COMPLETED_TOOLS=$(count completed todo items)
if [ $COMPLETED_TOOLS -ne $TOTAL_TOOLS ]; then
echo "ERROR: Testing incomplete. $COMPLETED_TOOLS/$TOTAL_TOOLS tested"
exit 1
fi
```
### CRITICAL: NO TOOL LEFT UNTESTED
- **Every tool name from the JSON list must be individually tested**
- **Every tool must have a TodoWrite entry that gets marked completed**
- **Tools that fail due to missing parameters should be tested anyway and marked as blocked**
- **Tools that require setup (like running processes) should be tested and documented as requiring dependencies**
- **NO ASSUMPTIONS**: Test tools even if they seem redundant or similar to others
### BLACK BOX TESTING ENFORCEMENT
- ✅ **Test only through Reloaderoo MCP interface** - Simulates real-world MCP client usage
- ✅ **Use task lists** - Track progress with TodoWrite tool for every single tool
- ✅ **Tick off each tool** - Mark completed in task list after manual verification
- ✅ **Manual oversight** - Human verification of each tool's input and output
- ❌ **Never examine source code** - No reading implementation files during testing
- ❌ **Never diagnose internal issues** - No investigation of build processes or tool registration
- ❌ **Never suggest implementation fixes** - Report issues as findings, don't solve them
- ❌ **Never use scripts for tool testing** - Each tool must be manually executed and verified
### 🚨 TESTING PSYCHOLOGY & BIAS PREVENTION
### COMMON ANTI-PATTERNS TO AVOID
#### 1. Efficiency Bias (FORBIDDEN)
- **Symptom**: "These tools look similar, I'll test one to validate the others"
- **Correction**: Every tool is unique and must be tested individually
- **Enforcement**: Count tools at start, verify same count tested at end
#### 2. Pattern Recognition Override (FORBIDDEN)
- **Symptom**: "I see the pattern, the rest will work the same way"
- **Correction**: Patterns may hide edge cases, bugs, or different implementations
- **Enforcement**: No assumptions allowed, test every tool regardless of apparent similarity
#### 3. Time Pressure Shortcuts (FORBIDDEN)
- **Symptom**: "This is taking too long, let me speed up by sampling"
- **Correction**: This is explicitly a long-running task with no time constraints
- **Enforcement**: Thoroughness is the ONLY priority, efficiency is irrelevant
#### 4. False Confidence (FORBIDDEN)
- **Symptom**: "The architecture is solid, so all tools must work"
- **Correction**: Architecture validation does not guarantee individual tool functionality
- **Enforcement**: Test tools to discover actual issues, not to confirm assumptions
### MANDATORY MINDSET
- **Every tool is potentially broken** until individually tested
- **Every tool may have unique edge cases** not covered by similar tools
- **Every tool deserves individual attention** regardless of apparent redundancy
- **Testing completion means EVERY tool tested**, not "enough tools to validate patterns"
- **The goal is discovering problems**, not confirming everything works
### TESTING COMPLETENESS CHECKLIST
- [ ] Generated complete tool list (83+ tools)
- [ ] Created TodoWrite entry for every single tool
- [ ] Tested every tool individually via Reloaderoo inspect
- [ ] Marked every tool as completed in TodoWrite
- [ ] Verified tool count: tested_count == total_count
- [ ] Documented all results, including failures and blocked tools
- [ ] Created final report covering ALL tools, not just successful ones
### Tool Dependency Graph Testing Strategy
**CRITICAL: Tools must be tested in dependency order:**
1. **Foundation Tools** (provide data for other tools):
- `doctor` - System info
- `list_devices` - Device UUIDs
- `list_sims` - Simulator UUIDs
- `discover_projs` - Project/workspace paths
2. **Discovery Tools** (provide metadata for build tools):
- `list_schemes` - Scheme names
- `show_build_settings` - Build settings
3. **Build Tools** (create artifacts for install tools):
- `build_*` tools - Create app bundles
- `get_*_app_path_*` tools - Locate built app bundles
- `get_*_bundle_id` tools - Extract bundle IDs
4. **Installation Tools** (depend on built artifacts):
- `install_app_*` tools - Install built apps
- `launch_app_*` tools - Launch installed apps
5. **Testing Tools** (depend on projects/schemes):
- `test_*` tools - Run test suites
6. **UI Automation Tools** (depend on running apps):
- `describe_ui`, `screenshot`, `tap`, etc.
### MANDATORY: Record Key Outputs
Must capture and document these values for dependent tools:
- **Device UUIDs** from `list_devices`
- **Simulator UUIDs** from `list_sims`
- **Project/workspace paths** from `discover_projs`
- **Scheme names** from `list_schems_*`
- **App bundle paths** from `get_*_app_path_*`
- **Bundle IDs** from `get_*_bundle_id`
### Prerequisites
1. **Build the server**: `npm run build`
2. **Install jq**: `brew install jq` (required for JSON parsing)
3. **System Requirements**: macOS with Xcode installed, connected devices/simulators optional
### Step 1: Programmatic Discovery and Official Testing Lists
#### Generate Official Tool List
```bash
# Generate complete tool list with accurate count
npx reloaderoo@latest inspect list-tools -- node build/index.js 2>/dev/null > /tmp/tools.json
# Get accurate tool count
TOOL_COUNT=$(jq '.tools | length' /tmp/tools.json)
echo "Official tool count: $TOOL_COUNT"
# Generate tool names list for testing checklist
jq -r '.tools[] | .name' /tmp/tools.json > /tmp/tool_names.txt
echo "Tool names saved to /tmp/tool_names.txt"
```
#### Generate Official Resource List
```bash
# Generate complete resource list
npx reloaderoo@latest inspect list-resources -- node build/index.js 2>/dev/null > /tmp/resources.json
# Get accurate resource count
RESOURCE_COUNT=$(jq '.resources | length' /tmp/resources.json)
echo "Official resource count: $RESOURCE_COUNT"
# Generate resource URIs for testing checklist
jq -r '.resources[] | .uri' /tmp/resources.json > /tmp/resource_uris.txt
echo "Resource URIs saved to /tmp/resource_uris.txt"
```
#### Create Tool Testing Checklist
```bash
# Generate markdown checklist from actual tool list
echo "# Official Tool Testing Checklist" > /tmp/tool_testing_checklist.md
echo "" >> /tmp/tool_testing_checklist.md
echo "Total Tools: $TOOL_COUNT" >> /tmp/tool_testing_checklist.md
echo "" >> /tmp/tool_testing_checklist.md
# Add each tool as unchecked item
while IFS= read -r tool_name; do
echo "- [ ] $tool_name" >> /tmp/tool_testing_checklist.md
done < /tmp/tool_names.txt
echo "Tool testing checklist created at /tmp/tool_testing_checklist.md"
```
#### Create Resource Testing Checklist
```bash
# Generate markdown checklist from actual resource list
echo "# Official Resource Testing Checklist" > /tmp/resource_testing_checklist.md
echo "" >> /tmp/resource_testing_checklist.md
echo "Total Resources: $RESOURCE_COUNT" >> /tmp/resource_testing_checklist.md
echo "" >> /tmp/resource_testing_checklist.md
# Add each resource as unchecked item
while IFS= read -r resource_uri; do
echo "- [ ] $resource_uri" >> /tmp/resource_testing_checklist.md
done < /tmp/resource_uris.txt
echo "Resource testing checklist created at /tmp/resource_testing_checklist.md"
```
### Step 2: Tool Schema Discovery for Parameter Testing
#### Extract Tool Schema Information
```bash
# Get schema for specific tool to understand required parameters
TOOL_NAME="list_devices"
jq --arg tool "$TOOL_NAME" '.tools[] | select(.name == $tool) | .inputSchema' /tmp/tools.json
# Get tool description for usage guidance
jq --arg tool "$TOOL_NAME" '.tools[] | select(.name == $tool) | .description' /tmp/tools.json
# Generate parameter template for tool testing
jq --arg tool "$TOOL_NAME" '.tools[] | select(.name == $tool) | .inputSchema.properties // {}' /tmp/tools.json
```
#### Batch Schema Extraction
```bash
# Create schema reference file for all tools
echo "# Tool Schema Reference" > /tmp/tool_schemas.md
echo "" >> /tmp/tool_schemas.md
while IFS= read -r tool_name; do
echo "## $tool_name" >> /tmp/tool_schemas.md
echo "" >> /tmp/tool_schemas.md
# Get description
description=$(jq -r --arg tool "$tool_name" '.tools[] | select(.name == $tool) | .description' /tmp/tools.json)
echo "**Description:** $description" >> /tmp/tool_schemas.md
echo "" >> /tmp/tool_schemas.md
# Get required parameters
required=$(jq -r --arg tool "$tool_name" '.tools[] | select(.name == $tool) | .inputSchema.required // [] | join(", ")' /tmp/tools.json)
if [ "$required" != "" ]; then
echo "**Required Parameters:** $required" >> /tmp/tool_schemas.md
else
echo "**Required Parameters:** None" >> /tmp/tool_schemas.md
fi
echo "" >> /tmp/tool_schemas.md
# Get all parameters
echo "**All Parameters:**" >> /tmp/tool_schemas.md
jq --arg tool "$tool_name" '.tools[] | select(.name == $tool) | .inputSchema.properties // {} | keys[]' /tmp/tools.json | while read param; do
echo "- $param" >> /tmp/tool_schemas.md
done
echo "" >> /tmp/tool_schemas.md
done < /tmp/tool_names.txt
echo "Tool schema reference created at /tmp/tool_schemas.md"
```
### Step 3: Manual Tool-by-Tool Testing
#### 🚨 CRITICAL: STEP-BY-STEP BLACK BOX TESTING PROCESS
### ABSOLUTE RULE: ALL TESTING MUST BE DONE MANUALLY, ONE TOOL AT A TIME USING RELOADEROO INSPECT
### SYSTEMATIC TESTING PROCESS
1. **Create TodoWrite Task List**
- Add all 83 tools to task list before starting
- Mark each tool as "pending" initially
- Update status to "in_progress" when testing begins
- Mark "completed" only after manual verification
2. **Test Each Tool Individually**
- Execute ONLY via `npx reloaderoo@latest inspect call-tool "TOOL_NAME" --params 'JSON' -- node build/index.js`
- Wait for complete response before proceeding to next tool
- Read and verify each tool's output manually
- Record key outputs (UUIDs, paths, schemes) for dependent tools
3. **Manual Verification Requirements**
- ✅ **Read each response** - Manually verify tool output makes sense
- ✅ **Check for errors** - Identify any tool failures or unexpected responses
- ✅ **Record UUIDs/paths** - Save outputs needed for dependent tools
- ✅ **Update task list** - Mark each tool complete after verification
- ✅ **Document issues** - Record any problems found during testing
4. **FORBIDDEN SHORTCUTS:**
- ❌ **NO SCRIPTS** - Scripts hide what's happening and prevent proper verification
- ❌ **NO AUTOMATION** - Every tool call must be manually executed and verified
- ❌ **NO BATCHING** - Cannot test multiple tools simultaneously
- ❌ **NO MCP DIRECT CALLS** - Only Reloaderoo inspect commands allowed
#### Phase 1: Infrastructure Validation
#### Manual Commands (execute individually):
```bash
# Test server connectivity
npx reloaderoo@latest inspect ping -- node build/index.js
# Get server information
npx reloaderoo@latest inspect server-info -- node build/index.js
# Verify tool count manually
npx reloaderoo@latest inspect list-tools -- node build/index.js 2>/dev/null | jq '.tools | length'
# Verify resource count manually
npx reloaderoo@latest inspect list-resources -- node build/index.js 2>/dev/null | jq '.resources | length'
```
#### Phase 2: Resource Testing
```bash
# Test each resource systematically
while IFS= read -r resource_uri; do
echo "Testing resource: $resource_uri"
npx reloaderoo@latest inspect read-resource "$resource_uri" -- node build/index.js 2>/dev/null
echo "---"
done < /tmp/resource_uris.txt
```
#### Phase 3: Foundation Tools (Data Collection)
### CRITICAL: Capture ALL key outputs for dependent tools
```bash
echo "=== FOUNDATION TOOL TESTING & DATA COLLECTION ==="
# 1. Test doctor (no dependencies)
echo "Testing doctor..."
npx reloaderoo@latest inspect call-tool "doctor" --params '{}' -- node build/index.js 2>/dev/null
# 2. Collect device data
echo "Collecting device UUIDs..."
npx reloaderoo@latest inspect call-tool "list_devices" --params '{}' -- node build/index.js 2>/dev/null > /tmp/devices_output.json
DEVICE_UUIDS=$(jq -r '.content[0].text' /tmp/devices_output.json | grep -E "UDID: [A-F0-9-]+" | sed 's/.*UDID: //' | head -2)
echo "Device UUIDs captured: $DEVICE_UUIDS"
# 3. Collect simulator data
echo "Collecting simulator UUIDs..."
npx reloaderoo@latest inspect call-tool "list_sims" --params '{}' -- node build/index.js 2>/dev/null > /tmp/sims_output.json
SIMULATOR_UUIDS=$(jq -r '.content[0].text' /tmp/sims_output.json | grep -E "\([A-F0-9-]+\)" | sed 's/.*(\([A-F0-9-]*\)).*/\1/' | head -3)
echo "Simulator UUIDs captured: $SIMULATOR_UUIDS"
# 4. Collect project data
echo "Collecting project paths..."
npx reloaderoo@latest inspect call-tool "discover_projs" --params '{"workspaceRoot": "/Volumes/Developer/XcodeBuildMCP"}' -- node build/index.js 2>/dev/null > /tmp/projects_output.json
PROJECT_PATHS=$(jq -r '.content[1].text' /tmp/projects_output.json | grep -E "\.xcodeproj$" | sed 's/.*- //' | head -3)
WORKSPACE_PATHS=$(jq -r '.content[2].text' /tmp/projects_output.json | grep -E "\.xcworkspace$" | sed 's/.*- //' | head -2)
echo "Project paths captured: $PROJECT_PATHS"
echo "Workspace paths captured: $WORKSPACE_PATHS"
# Save key data for dependent tools
echo "$DEVICE_UUIDS" > /tmp/device_uuids.txt
echo "$SIMULATOR_UUIDS" > /tmp/simulator_uuids.txt
echo "$PROJECT_PATHS" > /tmp/project_paths.txt
echo "$WORKSPACE_PATHS" > /tmp/workspace_paths.txt
```
#### Phase 4: Discovery Tools (Metadata Collection)
```bash
echo "=== DISCOVERY TOOL TESTING & METADATA COLLECTION ==="
# Collect schemes for each project
while IFS= read -r project_path; do
if [ -n "$project_path" ]; then
echo "Getting schemes for: $project_path"
npx reloaderoo@latest inspect call-tool "list_schems_proj" --params "{\"projectPath\": \"$project_path\"}" -- node build/index.js 2>/dev/null > /tmp/schemes_$$.json
SCHEMES=$(jq -r '.content[1].text' /tmp/schemes_$$.json 2>/dev/null || echo "NoScheme")
echo "$project_path|$SCHEMES" >> /tmp/project_schemes.txt
echo "Schemes captured for $project_path: $SCHEMES"
fi
done < /tmp/project_paths.txt
# Collect schemes for each workspace
while IFS= read -r workspace_path; do
if [ -n "$workspace_path" ]; then
echo "Getting schemes for: $workspace_path"
npx reloaderoo@latest inspect call-tool "list_schemes" --params "{\"workspacePath\": \"$workspace_path\"}" -- node build/index.js 2>/dev/null > /tmp/ws_schemes_$$.json
SCHEMES=$(jq -r '.content[1].text' /tmp/ws_schemes_$$.json 2>/dev/null || echo "NoScheme")
echo "$workspace_path|$SCHEMES" >> /tmp/workspace_schemes.txt
echo "Schemes captured for $workspace_path: $SCHEMES"
fi
done < /tmp/workspace_paths.txt
```
#### Phase 5: Manual Individual Tool Testing (All 83 Tools)
### CRITICAL: Test every single tool manually, one at a time
#### Manual Testing Process:
1. **Create task list** with TodoWrite tool for all 83 tools
2. **Test each tool individually** with proper parameters
3. **Mark each tool complete** in task list after manual verification
4. **Record results** and observations for each tool
5. **NO SCRIPTS** - Each command executed manually
### STEP-BY-STEP MANUAL TESTING COMMANDS
```bash
# STEP 1: Test foundation tools (no parameters required)
# Execute each command individually, wait for response, verify manually
npx reloaderoo@latest inspect call-tool "doctor" --params '{}' -- node build/index.js
# [Wait for response, read output, mark tool complete in task list]
npx reloaderoo@latest inspect call-tool "list_devices" --params '{}' -- node build/index.js
# [Record device UUIDs from response for dependent tools]
npx reloaderoo@latest inspect call-tool "list_sims" --params '{}' -- node build/index.js
# [Record simulator UUIDs from response for dependent tools]
# STEP 2: Test project discovery (use discovered project paths)
npx reloaderoo@latest inspect call-tool "list_schems_proj" --params '{"projectPath": "/actual/path/from/discover_projs.xcodeproj"}' -- node build/index.js
# [Record scheme names from response for build tools]
# STEP 3: Test workspace tools (use discovered workspace paths)
npx reloaderoo@latest inspect call-tool "list_schemes" --params '{"workspacePath": "/actual/path/from/discover_projs.xcworkspace"}' -- node build/index.js
# [Record scheme names from response for build tools]
# STEP 4: Test simulator tools (use captured simulator UUIDs from step 1)
npx reloaderoo@latest inspect call-tool "boot_sim" --params '{"simulatorUuid": "ACTUAL_UUID_FROM_LIST_SIMS"}' -- node build/index.js
# [Verify simulator boots successfully]
# STEP 5: Test build tools (requires project + scheme + simulator from previous steps)
npx reloaderoo@latest inspect call-tool "build_sim_id_proj" --params '{"projectPath": "/actual/project.xcodeproj", "scheme": "ActualSchemeName", "simulatorId": "ACTUAL_SIMULATOR_UUID"}' -- node build/index.js
# [Verify build succeeds and record app bundle path]
```
### CRITICAL: EACH COMMAND MUST BE
1. **Executed individually** - One command at a time, manually typed or pasted
2. **Verified manually** - Read the complete response before continuing
3. **Tracked in task list** - Mark tool complete only after verification
4. **Use real data** - Replace placeholder values with actual captured data
5. **Wait for completion** - Allow each command to finish before proceeding
### TESTING VIOLATIONS AND ENFORCEMENT
### 🚨 CRITICAL VIOLATIONS THAT WILL TERMINATE TESTING
1. **Direct MCP Tool Usage Violation:**
```typescript
// ❌ IMMEDIATE TERMINATION - Using MCP tools directly
await mcp__XcodeBuildMCP__list_devices();
const result = await list_sims();
```
2. **Script-Based Testing Violation:**
```bash
# ❌ IMMEDIATE TERMINATION - Using scripts to test tools
for tool in $(cat tool_list.txt); do
npx reloaderoo inspect call-tool "$tool" --params '{}' -- node build/index.js
done
```
3. **Batching/Automation Violation:**
```bash
# ❌ IMMEDIATE TERMINATION - Testing multiple tools simultaneously
npx reloaderoo inspect call-tool "list_devices" & npx reloaderoo inspect call-tool "list_sims" &
```
4. **Source Code Examination Violation:**
```typescript
// ❌ IMMEDIATE TERMINATION - Reading implementation during testing
const toolImplementation = await Read('/src/mcp/tools/device-shared/list_devices.ts');
```
### ENFORCEMENT PROCEDURE
1. **First Violation**: Immediate correction and restart of testing process
2. **Documentation Update**: Add explicit prohibition to prevent future violations
3. **Method Validation**: Ensure all future testing uses only Reloaderoo inspect commands
4. **Progress Reset**: Restart testing from foundation tools if direct MCP usage detected
### VALID TESTING SEQUENCE EXAMPLE
```bash
# ✅ CORRECT - Step-by-step manual execution via Reloaderoo
# Tool 1: Test doctor
npx reloaderoo@latest inspect call-tool "doctor" --params '{}' -- node build/index.js
# [Read response, verify, mark complete in TodoWrite]
# Tool 2: Test list_devices
npx reloaderoo@latest inspect call-tool "list_devices" --params '{}' -- node build/index.js
# [Read response, capture UUIDs, mark complete in TodoWrite]
# Tool 3: Test list_sims
npx reloaderoo@latest inspect call-tool "list_sims" --params '{}' -- node build/index.js
# [Read response, capture UUIDs, mark complete in TodoWrite]
# Tool X: Test stateful tool (expected to fail)
npx reloaderoo@latest inspect call-tool "swift_package_stop" --params '{"pid": 12345}' -- node build/index.js
# [Tool fails as expected - no in-memory state available]
# [Mark as "false negative - stateful tool limitation" in TodoWrite]
# [Continue to next tool without investigation]
# Continue individually for all 83 tools...
```
### HANDLING STATEFUL TOOL FAILURES
```bash
# ✅ CORRECT Response to Expected Stateful Tool Failure
# Tool fails with "No process found" or similar state-related error
# Response: Mark tool as "tested - false negative (stateful)" in task list
# Do NOT attempt to diagnose, fix, or investigate the failure
# Continue immediately to next tool in sequence
```
### Step 4: Error Testing
```bash
# Test error handling systematically
echo "=== Error Testing ==="
# Test with invalid JSON parameters
echo "Testing invalid parameter types..."
npx reloaderoo@latest inspect call-tool list_schems_proj --params '{"projectPath": 123}' -- node build/index.js 2>/dev/null
# Test with non-existent paths
echo "Testing non-existent paths..."
npx reloaderoo@latest inspect call-tool list_schems_proj --params '{"projectPath": "/nonexistent/path.xcodeproj"}' -- node build/index.js 2>/dev/null
# Test with invalid UUIDs
echo "Testing invalid UUIDs..."
npx reloaderoo@latest inspect call-tool boot_sim --params '{"simulatorUuid": "invalid-uuid"}' -- node build/index.js 2>/dev/null
```
### Step 5: Generate Testing Report
```bash
# Create comprehensive testing session report
cat > TESTING_SESSION_$(date +%Y-%m-%d).md << EOF
# Manual Testing Session - $(date +%Y-%m-%d)
## Environment
- macOS Version: $(sw_vers -productVersion)
- XcodeBuildMCP Version: $(jq -r '.version' package.json 2>/dev/null || echo "unknown")
- Testing Method: Reloaderoo @latest via npx
## Official Counts (Programmatically Verified)
- Total Tools: $TOOL_COUNT
- Total Resources: $RESOURCE_COUNT
## Test Results
[Document test results here]
## Issues Found
[Document any discrepancies or failures]
## Performance Notes
[Document response times and performance observations]
EOF
echo "Testing session template created: TESTING_SESSION_$(date +%Y-%m-%d).md"
```
### Key Commands Reference
```bash
# Essential testing commands
npx reloaderoo@latest inspect ping -- node build/index.js
npx reloaderoo@latest inspect server-info -- node build/index.js
npx reloaderoo@latest inspect list-tools -- node build/index.js | jq '.tools | length'
npx reloaderoo@latest inspect list-resources -- node build/index.js | jq '.resources | length'
npx reloaderoo@latest inspect call-tool TOOL_NAME --params '{}' -- node build/index.js
npx reloaderoo@latest inspect read-resource "xcodebuildmcp://RESOURCE" -- node build/index.js
# Schema extraction
jq --arg tool "TOOL_NAME" '.tools[] | select(.name == $tool) | .inputSchema' /tmp/tools.json
jq --arg tool "TOOL_NAME" '.tools[] | select(.name == $tool) | .description' /tmp/tools.json
```
This systematic approach ensures comprehensive, accurate testing using programmatic discovery and validation of all XcodeBuildMCP functionality.
## Troubleshooting
### Common Issues
#### 1. "Real System Executor Detected" Error
**Symptoms**: Test fails with error about real system executor being used
**Cause**: Handler not receiving mock executor parameter
**Fix**: Ensure test passes createMockExecutor() to handler:
```typescript
// ❌ WRONG
const result = await tool.handler(params);
// ✅ CORRECT
const mockExecutor = createMockExecutor({ success: true });
const result = await tool.handler(params, mockExecutor);
```
#### 2. "Real Filesystem Executor Detected" Error
**Symptoms**: Test fails when trying to access file system
**Cause**: Handler not receiving mock file system executor
**Fix**: Pass createMockFileSystemExecutor():
```typescript
const mockCmd = createMockExecutor({ success: true });
const mockFS = createMockFileSystemExecutor({ readFile: async () => 'content' });
const result = await tool.handler(params, mockCmd, mockFS);
```
#### 3. Handler Signature Errors
**Symptoms**: TypeScript errors about handler parameters
**Cause**: Handler doesn't support dependency injection
**Fix**: Update handler signature:
```typescript
async handler(args: Record<string, unknown>): Promise<ToolResponse> {
return tool_nameLogic(args, getDefaultCommandExecutor(), getDefaultFileSystemExecutor());
}
```
### Debug Commands
```bash
# Run specific test file
npm test -- src/plugins/simulator-workspace/__tests__/tool_name.test.ts
# Run with verbose output
npm test -- --reporter=verbose
# Check for banned patterns
node scripts/check-code-patterns.js
# Verify dependency injection compliance
node scripts/audit-dependency-container.js
# Coverage for specific directory
npm run test:coverage -- src/plugins/simulator-workspace/
```
### Validation Scripts
```bash
# Check for vitest mocking violations
node scripts/check-code-patterns.js --pattern=vitest
# Check dependency injection compliance
node scripts/audit-dependency-container.js
# Both scripts must pass before committing
```
## Best Practices Summary
1. **Dependency injection**: Always use createMockExecutor() and createMockFileSystemExecutor()
2. **No vitest mocking**: All vi.mock, vi.fn, etc. patterns are banned
3. **Three dimensions**: Test input validation, command execution, and output processing
4. **Literal expectations**: Use exact strings in assertions to catch regressions
5. **Performance**: Ensure fast execution through proper mocking
6. **Coverage**: Aim for 95%+ with focus on error paths
7. **Consistency**: Follow standard patterns across all plugin tests
8. **Test safety**: Default executors prevent accidental real system calls
This testing strategy ensures robust, maintainable tests that provide confidence in plugin functionality while remaining resilient to implementation changes and completely eliminating vitest mocking dependencies.
```