This is page 17 of 20. Use http://codebase.md/trycua/cua?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .all-contributorsrc
├── .cursorignore
├── .devcontainer
│ ├── devcontainer.json
│ ├── post-install.sh
│ └── README.md
├── .dockerignore
├── .gitattributes
├── .github
│ ├── FUNDING.yml
│ ├── scripts
│ │ ├── get_pyproject_version.py
│ │ └── tests
│ │ ├── __init__.py
│ │ ├── README.md
│ │ └── test_get_pyproject_version.py
│ └── workflows
│ ├── ci-lume.yml
│ ├── docker-publish-kasm.yml
│ ├── docker-publish-xfce.yml
│ ├── docker-reusable-publish.yml
│ ├── npm-publish-computer.yml
│ ├── npm-publish-core.yml
│ ├── publish-lume.yml
│ ├── pypi-publish-agent.yml
│ ├── pypi-publish-computer-server.yml
│ ├── pypi-publish-computer.yml
│ ├── pypi-publish-core.yml
│ ├── pypi-publish-mcp-server.yml
│ ├── pypi-publish-pylume.yml
│ ├── pypi-publish-som.yml
│ ├── pypi-reusable-publish.yml
│ └── test-validation-script.yml
├── .gitignore
├── .vscode
│ ├── docs.code-workspace
│ ├── launch.json
│ ├── libs-ts.code-workspace
│ ├── lume.code-workspace
│ ├── lumier.code-workspace
│ └── py.code-workspace
├── blog
│ ├── app-use.md
│ ├── assets
│ │ ├── composite-agents.png
│ │ ├── docker-ubuntu-support.png
│ │ ├── hack-booth.png
│ │ ├── hack-closing-ceremony.jpg
│ │ ├── hack-cua-ollama-hud.jpeg
│ │ ├── hack-leaderboard.png
│ │ ├── hack-the-north.png
│ │ ├── hack-winners.jpeg
│ │ ├── hack-workshop.jpeg
│ │ ├── hud-agent-evals.png
│ │ └── trajectory-viewer.jpeg
│ ├── bringing-computer-use-to-the-web.md
│ ├── build-your-own-operator-on-macos-1.md
│ ├── build-your-own-operator-on-macos-2.md
│ ├── composite-agents.md
│ ├── cua-hackathon.md
│ ├── hack-the-north.md
│ ├── hud-agent-evals.md
│ ├── human-in-the-loop.md
│ ├── introducing-cua-cloud-containers.md
│ ├── lume-to-containerization.md
│ ├── sandboxed-python-execution.md
│ ├── training-computer-use-models-trajectories-1.md
│ ├── trajectory-viewer.md
│ ├── ubuntu-docker-support.md
│ └── windows-sandbox.md
├── CONTRIBUTING.md
├── Development.md
├── Dockerfile
├── docs
│ ├── .gitignore
│ ├── .prettierrc
│ ├── content
│ │ └── docs
│ │ ├── agent-sdk
│ │ │ ├── agent-loops.mdx
│ │ │ ├── benchmarks
│ │ │ │ ├── index.mdx
│ │ │ │ ├── interactive.mdx
│ │ │ │ ├── introduction.mdx
│ │ │ │ ├── meta.json
│ │ │ │ ├── osworld-verified.mdx
│ │ │ │ ├── screenspot-pro.mdx
│ │ │ │ └── screenspot-v2.mdx
│ │ │ ├── callbacks
│ │ │ │ ├── agent-lifecycle.mdx
│ │ │ │ ├── cost-saving.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── logging.mdx
│ │ │ │ ├── meta.json
│ │ │ │ ├── pii-anonymization.mdx
│ │ │ │ └── trajectories.mdx
│ │ │ ├── chat-history.mdx
│ │ │ ├── custom-computer-handlers.mdx
│ │ │ ├── custom-tools.mdx
│ │ │ ├── customizing-computeragent.mdx
│ │ │ ├── integrations
│ │ │ │ ├── hud.mdx
│ │ │ │ └── meta.json
│ │ │ ├── message-format.mdx
│ │ │ ├── meta.json
│ │ │ ├── migration-guide.mdx
│ │ │ ├── prompt-caching.mdx
│ │ │ ├── supported-agents
│ │ │ │ ├── composed-agents.mdx
│ │ │ │ ├── computer-use-agents.mdx
│ │ │ │ ├── grounding-models.mdx
│ │ │ │ ├── human-in-the-loop.mdx
│ │ │ │ └── meta.json
│ │ │ ├── supported-model-providers
│ │ │ │ ├── index.mdx
│ │ │ │ └── local-models.mdx
│ │ │ └── usage-tracking.mdx
│ │ ├── computer-sdk
│ │ │ ├── commands.mdx
│ │ │ ├── computer-ui.mdx
│ │ │ ├── computers.mdx
│ │ │ ├── meta.json
│ │ │ └── sandboxed-python.mdx
│ │ ├── index.mdx
│ │ ├── libraries
│ │ │ ├── agent
│ │ │ │ └── index.mdx
│ │ │ ├── computer
│ │ │ │ └── index.mdx
│ │ │ ├── computer-server
│ │ │ │ ├── Commands.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── REST-API.mdx
│ │ │ │ └── WebSocket-API.mdx
│ │ │ ├── core
│ │ │ │ └── index.mdx
│ │ │ ├── lume
│ │ │ │ ├── cli-reference.mdx
│ │ │ │ ├── faq.md
│ │ │ │ ├── http-api.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── installation.mdx
│ │ │ │ ├── meta.json
│ │ │ │ └── prebuilt-images.mdx
│ │ │ ├── lumier
│ │ │ │ ├── building-lumier.mdx
│ │ │ │ ├── docker-compose.mdx
│ │ │ │ ├── docker.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── installation.mdx
│ │ │ │ └── meta.json
│ │ │ ├── mcp-server
│ │ │ │ ├── client-integrations.mdx
│ │ │ │ ├── configuration.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── installation.mdx
│ │ │ │ ├── llm-integrations.mdx
│ │ │ │ ├── meta.json
│ │ │ │ ├── tools.mdx
│ │ │ │ └── usage.mdx
│ │ │ └── som
│ │ │ ├── configuration.mdx
│ │ │ └── index.mdx
│ │ ├── meta.json
│ │ ├── quickstart-cli.mdx
│ │ ├── quickstart-devs.mdx
│ │ └── telemetry.mdx
│ ├── next.config.mjs
│ ├── package-lock.json
│ ├── package.json
│ ├── pnpm-lock.yaml
│ ├── postcss.config.mjs
│ ├── public
│ │ └── img
│ │ ├── agent_gradio_ui.png
│ │ ├── agent.png
│ │ ├── cli.png
│ │ ├── computer.png
│ │ ├── som_box_threshold.png
│ │ └── som_iou_threshold.png
│ ├── README.md
│ ├── source.config.ts
│ ├── src
│ │ ├── app
│ │ │ ├── (home)
│ │ │ │ ├── [[...slug]]
│ │ │ │ │ └── page.tsx
│ │ │ │ └── layout.tsx
│ │ │ ├── api
│ │ │ │ └── search
│ │ │ │ └── route.ts
│ │ │ ├── favicon.ico
│ │ │ ├── global.css
│ │ │ ├── layout.config.tsx
│ │ │ ├── layout.tsx
│ │ │ ├── llms.mdx
│ │ │ │ └── [[...slug]]
│ │ │ │ └── route.ts
│ │ │ └── llms.txt
│ │ │ └── route.ts
│ │ ├── assets
│ │ │ ├── discord-black.svg
│ │ │ ├── discord-white.svg
│ │ │ ├── logo-black.svg
│ │ │ └── logo-white.svg
│ │ ├── components
│ │ │ ├── iou.tsx
│ │ │ └── mermaid.tsx
│ │ ├── lib
│ │ │ ├── llms.ts
│ │ │ └── source.ts
│ │ └── mdx-components.tsx
│ └── tsconfig.json
├── examples
│ ├── agent_examples.py
│ ├── agent_ui_examples.py
│ ├── computer_examples_windows.py
│ ├── computer_examples.py
│ ├── computer_ui_examples.py
│ ├── computer-example-ts
│ │ ├── .env.example
│ │ ├── .gitignore
│ │ ├── .prettierrc
│ │ ├── package-lock.json
│ │ ├── package.json
│ │ ├── pnpm-lock.yaml
│ │ ├── README.md
│ │ ├── src
│ │ │ ├── helpers.ts
│ │ │ └── index.ts
│ │ └── tsconfig.json
│ ├── docker_examples.py
│ ├── evals
│ │ ├── hud_eval_examples.py
│ │ └── wikipedia_most_linked.txt
│ ├── pylume_examples.py
│ ├── sandboxed_functions_examples.py
│ ├── som_examples.py
│ ├── utils.py
│ └── winsandbox_example.py
├── img
│ ├── agent_gradio_ui.png
│ ├── agent.png
│ ├── cli.png
│ ├── computer.png
│ ├── logo_black.png
│ └── logo_white.png
├── libs
│ ├── kasm
│ │ ├── Dockerfile
│ │ ├── LICENSE
│ │ ├── README.md
│ │ └── src
│ │ └── ubuntu
│ │ └── install
│ │ └── firefox
│ │ ├── custom_startup.sh
│ │ ├── firefox.desktop
│ │ └── install_firefox.sh
│ ├── lume
│ │ ├── .cursorignore
│ │ ├── CONTRIBUTING.md
│ │ ├── Development.md
│ │ ├── img
│ │ │ └── cli.png
│ │ ├── Package.resolved
│ │ ├── Package.swift
│ │ ├── README.md
│ │ ├── resources
│ │ │ └── lume.entitlements
│ │ ├── scripts
│ │ │ ├── build
│ │ │ │ ├── build-debug.sh
│ │ │ │ ├── build-release-notarized.sh
│ │ │ │ └── build-release.sh
│ │ │ └── install.sh
│ │ ├── src
│ │ │ ├── Commands
│ │ │ │ ├── Clone.swift
│ │ │ │ ├── Config.swift
│ │ │ │ ├── Create.swift
│ │ │ │ ├── Delete.swift
│ │ │ │ ├── Get.swift
│ │ │ │ ├── Images.swift
│ │ │ │ ├── IPSW.swift
│ │ │ │ ├── List.swift
│ │ │ │ ├── Logs.swift
│ │ │ │ ├── Options
│ │ │ │ │ └── FormatOption.swift
│ │ │ │ ├── Prune.swift
│ │ │ │ ├── Pull.swift
│ │ │ │ ├── Push.swift
│ │ │ │ ├── Run.swift
│ │ │ │ ├── Serve.swift
│ │ │ │ ├── Set.swift
│ │ │ │ └── Stop.swift
│ │ │ ├── ContainerRegistry
│ │ │ │ ├── ImageContainerRegistry.swift
│ │ │ │ ├── ImageList.swift
│ │ │ │ └── ImagesPrinter.swift
│ │ │ ├── Errors
│ │ │ │ └── Errors.swift
│ │ │ ├── FileSystem
│ │ │ │ ├── Home.swift
│ │ │ │ ├── Settings.swift
│ │ │ │ ├── VMConfig.swift
│ │ │ │ ├── VMDirectory.swift
│ │ │ │ └── VMLocation.swift
│ │ │ ├── LumeController.swift
│ │ │ ├── Main.swift
│ │ │ ├── Server
│ │ │ │ ├── Handlers.swift
│ │ │ │ ├── HTTP.swift
│ │ │ │ ├── Requests.swift
│ │ │ │ ├── Responses.swift
│ │ │ │ └── Server.swift
│ │ │ ├── Utils
│ │ │ │ ├── CommandRegistry.swift
│ │ │ │ ├── CommandUtils.swift
│ │ │ │ ├── Logger.swift
│ │ │ │ ├── NetworkUtils.swift
│ │ │ │ ├── Path.swift
│ │ │ │ ├── ProcessRunner.swift
│ │ │ │ ├── ProgressLogger.swift
│ │ │ │ ├── String.swift
│ │ │ │ └── Utils.swift
│ │ │ ├── Virtualization
│ │ │ │ ├── DarwinImageLoader.swift
│ │ │ │ ├── DHCPLeaseParser.swift
│ │ │ │ ├── ImageLoaderFactory.swift
│ │ │ │ └── VMVirtualizationService.swift
│ │ │ ├── VM
│ │ │ │ ├── DarwinVM.swift
│ │ │ │ ├── LinuxVM.swift
│ │ │ │ ├── VM.swift
│ │ │ │ ├── VMDetails.swift
│ │ │ │ ├── VMDetailsPrinter.swift
│ │ │ │ ├── VMDisplayResolution.swift
│ │ │ │ └── VMFactory.swift
│ │ │ └── VNC
│ │ │ ├── PassphraseGenerator.swift
│ │ │ └── VNCService.swift
│ │ └── tests
│ │ ├── Mocks
│ │ │ ├── MockVM.swift
│ │ │ ├── MockVMVirtualizationService.swift
│ │ │ └── MockVNCService.swift
│ │ ├── VM
│ │ │ └── VMDetailsPrinterTests.swift
│ │ ├── VMTests.swift
│ │ ├── VMVirtualizationServiceTests.swift
│ │ └── VNCServiceTests.swift
│ ├── lumier
│ │ ├── .dockerignore
│ │ ├── Dockerfile
│ │ ├── README.md
│ │ └── src
│ │ ├── bin
│ │ │ └── entry.sh
│ │ ├── config
│ │ │ └── constants.sh
│ │ ├── hooks
│ │ │ └── on-logon.sh
│ │ └── lib
│ │ ├── utils.sh
│ │ └── vm.sh
│ ├── python
│ │ ├── agent
│ │ │ ├── agent
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __main__.py
│ │ │ │ ├── adapters
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── huggingfacelocal_adapter.py
│ │ │ │ │ ├── human_adapter.py
│ │ │ │ │ ├── mlxvlm_adapter.py
│ │ │ │ │ └── models
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── generic.py
│ │ │ │ │ ├── internvl.py
│ │ │ │ │ ├── opencua.py
│ │ │ │ │ └── qwen2_5_vl.py
│ │ │ │ ├── agent.py
│ │ │ │ ├── callbacks
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── budget_manager.py
│ │ │ │ │ ├── image_retention.py
│ │ │ │ │ ├── logging.py
│ │ │ │ │ ├── operator_validator.py
│ │ │ │ │ ├── pii_anonymization.py
│ │ │ │ │ ├── prompt_instructions.py
│ │ │ │ │ ├── telemetry.py
│ │ │ │ │ └── trajectory_saver.py
│ │ │ │ ├── cli.py
│ │ │ │ ├── computers
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── cua.py
│ │ │ │ │ └── custom.py
│ │ │ │ ├── decorators.py
│ │ │ │ ├── human_tool
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── __main__.py
│ │ │ │ │ ├── server.py
│ │ │ │ │ └── ui.py
│ │ │ │ ├── integrations
│ │ │ │ │ └── hud
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── agent.py
│ │ │ │ │ └── proxy.py
│ │ │ │ ├── loops
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── anthropic.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── composed_grounded.py
│ │ │ │ │ ├── glm45v.py
│ │ │ │ │ ├── gta1.py
│ │ │ │ │ ├── holo.py
│ │ │ │ │ ├── internvl.py
│ │ │ │ │ ├── model_types.csv
│ │ │ │ │ ├── moondream3.py
│ │ │ │ │ ├── omniparser.py
│ │ │ │ │ ├── openai.py
│ │ │ │ │ ├── opencua.py
│ │ │ │ │ └── uitars.py
│ │ │ │ ├── proxy
│ │ │ │ │ ├── examples.py
│ │ │ │ │ └── handlers.py
│ │ │ │ ├── responses.py
│ │ │ │ ├── types.py
│ │ │ │ └── ui
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __main__.py
│ │ │ │ └── gradio
│ │ │ │ ├── __init__.py
│ │ │ │ ├── app.py
│ │ │ │ └── ui_components.py
│ │ │ ├── benchmarks
│ │ │ │ ├── .gitignore
│ │ │ │ ├── contrib.md
│ │ │ │ ├── interactive.py
│ │ │ │ ├── models
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ └── gta1.py
│ │ │ │ ├── README.md
│ │ │ │ ├── ss-pro.py
│ │ │ │ ├── ss-v2.py
│ │ │ │ └── utils.py
│ │ │ ├── example.py
│ │ │ ├── pyproject.toml
│ │ │ └── README.md
│ │ ├── computer
│ │ │ ├── computer
│ │ │ │ ├── __init__.py
│ │ │ │ ├── computer.py
│ │ │ │ ├── diorama_computer.py
│ │ │ │ ├── helpers.py
│ │ │ │ ├── interface
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── factory.py
│ │ │ │ │ ├── generic.py
│ │ │ │ │ ├── linux.py
│ │ │ │ │ ├── macos.py
│ │ │ │ │ ├── models.py
│ │ │ │ │ └── windows.py
│ │ │ │ ├── logger.py
│ │ │ │ ├── models.py
│ │ │ │ ├── providers
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── cloud
│ │ │ │ │ │ ├── __init__.py
│ │ │ │ │ │ └── provider.py
│ │ │ │ │ ├── docker
│ │ │ │ │ │ ├── __init__.py
│ │ │ │ │ │ └── provider.py
│ │ │ │ │ ├── factory.py
│ │ │ │ │ ├── lume
│ │ │ │ │ │ ├── __init__.py
│ │ │ │ │ │ └── provider.py
│ │ │ │ │ ├── lume_api.py
│ │ │ │ │ ├── lumier
│ │ │ │ │ │ ├── __init__.py
│ │ │ │ │ │ └── provider.py
│ │ │ │ │ └── winsandbox
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── provider.py
│ │ │ │ │ └── setup_script.ps1
│ │ │ │ ├── ui
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── __main__.py
│ │ │ │ │ └── gradio
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── app.py
│ │ │ │ └── utils.py
│ │ │ ├── poetry.toml
│ │ │ ├── pyproject.toml
│ │ │ └── README.md
│ │ ├── computer-server
│ │ │ ├── computer_server
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __main__.py
│ │ │ │ ├── cli.py
│ │ │ │ ├── diorama
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── diorama_computer.py
│ │ │ │ │ ├── diorama.py
│ │ │ │ │ ├── draw.py
│ │ │ │ │ ├── macos.py
│ │ │ │ │ └── safezone.py
│ │ │ │ ├── handlers
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── factory.py
│ │ │ │ │ ├── generic.py
│ │ │ │ │ ├── linux.py
│ │ │ │ │ ├── macos.py
│ │ │ │ │ └── windows.py
│ │ │ │ ├── main.py
│ │ │ │ ├── server.py
│ │ │ │ └── watchdog.py
│ │ │ ├── examples
│ │ │ │ ├── __init__.py
│ │ │ │ └── usage_example.py
│ │ │ ├── pyproject.toml
│ │ │ ├── README.md
│ │ │ ├── run_server.py
│ │ │ └── test_connection.py
│ │ ├── core
│ │ │ ├── core
│ │ │ │ ├── __init__.py
│ │ │ │ └── telemetry
│ │ │ │ ├── __init__.py
│ │ │ │ └── posthog.py
│ │ │ ├── poetry.toml
│ │ │ ├── pyproject.toml
│ │ │ └── README.md
│ │ ├── mcp-server
│ │ │ ├── mcp_server
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __main__.py
│ │ │ │ └── server.py
│ │ │ ├── pyproject.toml
│ │ │ ├── README.md
│ │ │ └── scripts
│ │ │ ├── install_mcp_server.sh
│ │ │ └── start_mcp_server.sh
│ │ ├── pylume
│ │ │ ├── __init__.py
│ │ │ ├── pylume
│ │ │ │ ├── __init__.py
│ │ │ │ ├── client.py
│ │ │ │ ├── exceptions.py
│ │ │ │ ├── lume
│ │ │ │ ├── models.py
│ │ │ │ ├── pylume.py
│ │ │ │ └── server.py
│ │ │ ├── pyproject.toml
│ │ │ └── README.md
│ │ └── som
│ │ ├── LICENSE
│ │ ├── poetry.toml
│ │ ├── pyproject.toml
│ │ ├── README.md
│ │ ├── som
│ │ │ ├── __init__.py
│ │ │ ├── detect.py
│ │ │ ├── detection.py
│ │ │ ├── models.py
│ │ │ ├── ocr.py
│ │ │ ├── util
│ │ │ │ └── utils.py
│ │ │ └── visualization.py
│ │ └── tests
│ │ └── test_omniparser.py
│ ├── typescript
│ │ ├── .gitignore
│ │ ├── .nvmrc
│ │ ├── agent
│ │ │ ├── examples
│ │ │ │ ├── playground-example.html
│ │ │ │ └── README.md
│ │ │ ├── package.json
│ │ │ ├── README.md
│ │ │ ├── src
│ │ │ │ ├── client.ts
│ │ │ │ ├── index.ts
│ │ │ │ └── types.ts
│ │ │ ├── tests
│ │ │ │ └── client.test.ts
│ │ │ ├── tsconfig.json
│ │ │ ├── tsdown.config.ts
│ │ │ └── vitest.config.ts
│ │ ├── biome.json
│ │ ├── computer
│ │ │ ├── .editorconfig
│ │ │ ├── .gitattributes
│ │ │ ├── .gitignore
│ │ │ ├── LICENSE
│ │ │ ├── package.json
│ │ │ ├── README.md
│ │ │ ├── src
│ │ │ │ ├── computer
│ │ │ │ │ ├── index.ts
│ │ │ │ │ ├── providers
│ │ │ │ │ │ ├── base.ts
│ │ │ │ │ │ ├── cloud.ts
│ │ │ │ │ │ └── index.ts
│ │ │ │ │ └── types.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── interface
│ │ │ │ │ ├── base.ts
│ │ │ │ │ ├── factory.ts
│ │ │ │ │ ├── index.ts
│ │ │ │ │ ├── linux.ts
│ │ │ │ │ ├── macos.ts
│ │ │ │ │ └── windows.ts
│ │ │ │ └── types.ts
│ │ │ ├── tests
│ │ │ │ ├── computer
│ │ │ │ │ └── cloud.test.ts
│ │ │ │ ├── interface
│ │ │ │ │ ├── factory.test.ts
│ │ │ │ │ ├── index.test.ts
│ │ │ │ │ ├── linux.test.ts
│ │ │ │ │ ├── macos.test.ts
│ │ │ │ │ └── windows.test.ts
│ │ │ │ └── setup.ts
│ │ │ ├── tsconfig.json
│ │ │ ├── tsdown.config.ts
│ │ │ └── vitest.config.ts
│ │ ├── core
│ │ │ ├── .editorconfig
│ │ │ ├── .gitattributes
│ │ │ ├── .gitignore
│ │ │ ├── LICENSE
│ │ │ ├── package.json
│ │ │ ├── README.md
│ │ │ ├── src
│ │ │ │ ├── index.ts
│ │ │ │ └── telemetry
│ │ │ │ ├── clients
│ │ │ │ │ ├── index.ts
│ │ │ │ │ └── posthog.ts
│ │ │ │ └── index.ts
│ │ │ ├── tests
│ │ │ │ └── telemetry.test.ts
│ │ │ ├── tsconfig.json
│ │ │ ├── tsdown.config.ts
│ │ │ └── vitest.config.ts
│ │ ├── package.json
│ │ ├── pnpm-lock.yaml
│ │ ├── pnpm-workspace.yaml
│ │ └── README.md
│ └── xfce
│ ├── .dockerignore
│ ├── .gitignore
│ ├── Dockerfile
│ ├── README.md
│ └── src
│ ├── scripts
│ │ ├── resize-display.sh
│ │ ├── start-computer-server.sh
│ │ ├── start-novnc.sh
│ │ ├── start-vnc.sh
│ │ └── xstartup.sh
│ ├── supervisor
│ │ └── supervisord.conf
│ └── xfce-config
│ ├── helpers.rc
│ ├── xfce4-power-manager.xml
│ └── xfce4-session.xml
├── LICENSE.md
├── notebooks
│ ├── agent_nb.ipynb
│ ├── blog
│ │ ├── build-your-own-operator-on-macos-1.ipynb
│ │ └── build-your-own-operator-on-macos-2.ipynb
│ ├── composite_agents_docker_nb.ipynb
│ ├── computer_nb.ipynb
│ ├── computer_server_nb.ipynb
│ ├── customizing_computeragent.ipynb
│ ├── eval_osworld.ipynb
│ ├── ollama_nb.ipynb
│ ├── pylume_nb.ipynb
│ ├── README.md
│ ├── sota_hackathon_cloud.ipynb
│ └── sota_hackathon.ipynb
├── pdm.lock
├── pyproject.toml
├── pyrightconfig.json
├── README.md
├── samples
│ └── community
│ ├── global-online
│ │ └── README.md
│ └── hack-the-north
│ └── README.md
├── scripts
│ ├── build-uv.sh
│ ├── build.ps1
│ ├── build.sh
│ ├── cleanup.sh
│ ├── playground-docker.sh
│ ├── playground.sh
│ └── run-docker-dev.sh
└── tests
├── pytest.ini
├── shell_cmd.py
├── test_files.py
├── test_shell_bash.py
├── test_telemetry.py
├── test_venv.py
└── test_watchdog.py
```
# Files
--------------------------------------------------------------------------------
/libs/lume/src/LumeController.swift:
--------------------------------------------------------------------------------
```swift
1 | import ArgumentParser
2 | import Foundation
3 | import Virtualization
4 |
5 | // MARK: - Shared VM Manager
6 |
7 | @MainActor
8 | final class SharedVM {
9 | static let shared: SharedVM = SharedVM()
10 | private var runningVMs: [String: VM] = [:]
11 |
12 | private init() {}
13 |
14 | func getVM(name: String) -> VM? {
15 | return runningVMs[name]
16 | }
17 |
18 | func setVM(name: String, vm: VM) {
19 | runningVMs[name] = vm
20 | }
21 |
22 | func removeVM(name: String) {
23 | runningVMs.removeValue(forKey: name)
24 | }
25 | }
26 |
27 | /// Entrypoint for Commands and API server
28 | final class LumeController {
29 | // MARK: - Properties
30 |
31 | let home: Home
32 | private let imageLoaderFactory: ImageLoaderFactory
33 | private let vmFactory: VMFactory
34 |
35 | // MARK: - Initialization
36 |
37 | init(
38 | home: Home = Home(),
39 | imageLoaderFactory: ImageLoaderFactory = DefaultImageLoaderFactory(),
40 | vmFactory: VMFactory = DefaultVMFactory()
41 | ) {
42 | self.home = home
43 | self.imageLoaderFactory = imageLoaderFactory
44 | self.vmFactory = vmFactory
45 | }
46 |
47 | // MARK: - Public VM Management Methods
48 |
49 | /// Lists all virtual machines in the system
50 | @MainActor
51 | public func list(storage: String? = nil) throws -> [VMDetails] {
52 | do {
53 | if let storage = storage {
54 | // If storage is specified, only return VMs from that location
55 | if storage.contains("/") || storage.contains("\\") {
56 | // Direct path - check if it exists
57 | if !FileManager.default.fileExists(atPath: storage) {
58 | // Return empty array if the path doesn't exist
59 | return []
60 | }
61 |
62 | // Try to get all VMs from the specified path
63 | // We need to check which subdirectories are valid VM dirs
64 | let directoryURL = URL(fileURLWithPath: storage)
65 | let contents = try FileManager.default.contentsOfDirectory(
66 | at: directoryURL,
67 | includingPropertiesForKeys: [.isDirectoryKey],
68 | options: .skipsHiddenFiles
69 | )
70 |
71 | let statuses = try contents.compactMap { subdir -> VMDetails? in
72 | guard let isDirectory = try subdir.resourceValues(forKeys: [.isDirectoryKey]).isDirectory,
73 | isDirectory else {
74 | return nil
75 | }
76 |
77 | let vmName = subdir.lastPathComponent
78 | // Check if it's a valid VM directory
79 | let vmDir = try home.getVMDirectoryFromPath(vmName, storagePath: storage)
80 | if !vmDir.initialized() {
81 | return nil
82 | }
83 |
84 | do {
85 | let vm = try self.get(name: vmName, storage: storage)
86 | return vm.details
87 | } catch {
88 | // Skip invalid VM directories
89 | return nil
90 | }
91 | }
92 | return statuses
93 | } else {
94 | // Named storage
95 | let vmsWithLoc = try home.getAllVMDirectories()
96 | let statuses = try vmsWithLoc.compactMap { vmWithLoc -> VMDetails? in
97 | // Only include VMs from the specified location
98 | if vmWithLoc.locationName != storage {
99 | return nil
100 | }
101 | let vm = try self.get(
102 | name: vmWithLoc.directory.name, storage: vmWithLoc.locationName)
103 | return vm.details
104 | }
105 | return statuses
106 | }
107 | } else {
108 | // No storage filter - get all VMs
109 | let vmsWithLoc = try home.getAllVMDirectories()
110 | let statuses = try vmsWithLoc.compactMap { vmWithLoc -> VMDetails? in
111 | let vm = try self.get(
112 | name: vmWithLoc.directory.name, storage: vmWithLoc.locationName)
113 | return vm.details
114 | }
115 | return statuses
116 | }
117 | } catch {
118 | Logger.error("Failed to list VMs", metadata: ["error": error.localizedDescription])
119 | throw error
120 | }
121 | }
122 |
123 | @MainActor
124 | public func clone(
125 | name: String, newName: String, sourceLocation: String? = nil, destLocation: String? = nil
126 | ) throws {
127 | let normalizedName = normalizeVMName(name: name)
128 | let normalizedNewName = normalizeVMName(name: newName)
129 | Logger.info(
130 | "Cloning VM",
131 | metadata: [
132 | "source": normalizedName,
133 | "destination": normalizedNewName,
134 | "sourceLocation": sourceLocation ?? "default",
135 | "destLocation": destLocation ?? "default",
136 | ])
137 |
138 | do {
139 | // Validate source VM exists
140 | _ = try self.validateVMExists(normalizedName, storage: sourceLocation)
141 |
142 | // Get the source VM and check if it's running
143 | let sourceVM = try get(name: normalizedName, storage: sourceLocation)
144 | if sourceVM.details.status == "running" {
145 | Logger.error("Cannot clone a running VM", metadata: ["source": normalizedName])
146 | throw VMError.alreadyRunning(normalizedName)
147 | }
148 |
149 | // Check if destination already exists
150 | do {
151 | let destDir = try home.getVMDirectory(normalizedNewName, storage: destLocation)
152 | if destDir.exists() {
153 | Logger.error(
154 | "Destination VM already exists",
155 | metadata: ["destination": normalizedNewName])
156 | throw HomeError.directoryAlreadyExists(path: destDir.dir.path)
157 | }
158 | } catch VMLocationError.locationNotFound {
159 | // Location not found is okay, we'll create it
160 | } catch VMError.notFound {
161 | // VM not found is okay, we'll create it
162 | }
163 |
164 | // Copy the VM directory
165 | try home.copyVMDirectory(
166 | from: normalizedName,
167 | to: normalizedNewName,
168 | sourceLocation: sourceLocation,
169 | destLocation: destLocation
170 | )
171 |
172 | // Update MAC address in the cloned VM to ensure uniqueness
173 | let clonedVM = try get(name: normalizedNewName, storage: destLocation)
174 | try clonedVM.setMacAddress(VZMACAddress.randomLocallyAdministered().string)
175 |
176 | // Update MAC Identifier in the cloned VM to ensure uniqueness
177 | try clonedVM.setMachineIdentifier(
178 | DarwinVirtualizationService.generateMachineIdentifier())
179 |
180 | Logger.info(
181 | "VM cloned successfully",
182 | metadata: ["source": normalizedName, "destination": normalizedNewName])
183 | } catch {
184 | Logger.error("Failed to clone VM", metadata: ["error": error.localizedDescription])
185 | throw error
186 | }
187 | }
188 |
189 | @MainActor
190 | public func get(name: String, storage: String? = nil) throws -> VM {
191 | let normalizedName = normalizeVMName(name: name)
192 | do {
193 | let vm: VM
194 | if let storagePath = storage, storagePath.contains("/") || storagePath.contains("\\") {
195 | // Storage is a direct path
196 | let vmDir = try home.getVMDirectoryFromPath(normalizedName, storagePath: storagePath)
197 | guard vmDir.initialized() else {
198 | // Throw a specific error if the directory exists but isn't a valid VM
199 | if vmDir.exists() {
200 | throw VMError.notInitialized(normalizedName)
201 | } else {
202 | throw VMError.notFound(normalizedName)
203 | }
204 | }
205 | // Pass the path as the storage context
206 | vm = try self.loadVM(vmDir: vmDir, storage: storagePath)
207 | } else {
208 | // Storage is nil or a named location
209 | let actualLocation = try self.validateVMExists(
210 | normalizedName, storage: storage)
211 |
212 | let vmDir = try home.getVMDirectory(normalizedName, storage: actualLocation)
213 | // loadVM will re-check initialized, but good practice to keep validateVMExists result.
214 | vm = try self.loadVM(vmDir: vmDir, storage: actualLocation)
215 | }
216 | return vm
217 | } catch {
218 | Logger.error(
219 | "Failed to get VM",
220 | metadata: [
221 | "vmName": normalizedName, "storage": storage ?? "default",
222 | "error": error.localizedDescription,
223 | ])
224 | // Re-throw the original error to preserve its type
225 | throw error
226 | }
227 | }
228 |
229 | @MainActor
230 | public func create(
231 | name: String,
232 | os: String,
233 | diskSize: UInt64,
234 | cpuCount: Int,
235 | memorySize: UInt64,
236 | display: String,
237 | ipsw: String?,
238 | storage: String? = nil
239 | ) async throws {
240 | Logger.info(
241 | "Creating VM",
242 | metadata: [
243 | "name": name,
244 | "os": os,
245 | "location": storage ?? "default",
246 | "disk_size": "\(diskSize / 1024 / 1024)MB",
247 | "cpu_count": "\(cpuCount)",
248 | "memory_size": "\(memorySize / 1024 / 1024)MB",
249 | "display": display,
250 | "ipsw": ipsw ?? "none",
251 | ])
252 |
253 | do {
254 | try validateCreateParameters(name: name, os: os, ipsw: ipsw, storage: storage)
255 |
256 | let vm = try await createTempVMConfig(
257 | os: os,
258 | cpuCount: cpuCount,
259 | memorySize: memorySize,
260 | diskSize: diskSize,
261 | display: display
262 | )
263 |
264 | try await vm.setup(
265 | ipswPath: ipsw ?? "none",
266 | cpuCount: cpuCount,
267 | memorySize: memorySize,
268 | diskSize: diskSize,
269 | display: display
270 | )
271 |
272 | try vm.finalize(to: name, home: home, storage: storage)
273 |
274 | Logger.info("VM created successfully", metadata: ["name": name])
275 | } catch {
276 | Logger.error("Failed to create VM", metadata: ["error": error.localizedDescription])
277 | throw error
278 | }
279 | }
280 |
281 | @MainActor
282 | public func delete(name: String, storage: String? = nil) async throws {
283 | let normalizedName = normalizeVMName(name: name)
284 | Logger.info(
285 | "Deleting VM",
286 | metadata: [
287 | "name": normalizedName,
288 | "location": storage ?? "default",
289 | ])
290 |
291 | do {
292 | let vmDir: VMDirectory
293 |
294 | // Check if storage is a direct path
295 | if let storagePath = storage, storagePath.contains("/") || storagePath.contains("\\") {
296 | // Storage is a direct path
297 | vmDir = try home.getVMDirectoryFromPath(normalizedName, storagePath: storagePath)
298 | guard vmDir.initialized() else {
299 | // Throw a specific error if the directory exists but isn't a valid VM
300 | if vmDir.exists() {
301 | throw VMError.notInitialized(normalizedName)
302 | } else {
303 | throw VMError.notFound(normalizedName)
304 | }
305 | }
306 | } else {
307 | // Storage is nil or a named location
308 | let actualLocation = try self.validateVMExists(normalizedName, storage: storage)
309 | vmDir = try home.getVMDirectory(normalizedName, storage: actualLocation)
310 | }
311 |
312 | // Stop VM if it's running
313 | if SharedVM.shared.getVM(name: normalizedName) != nil {
314 | try await stopVM(name: normalizedName)
315 | }
316 |
317 | try vmDir.delete()
318 |
319 | Logger.info("VM deleted successfully", metadata: ["name": normalizedName])
320 |
321 | } catch {
322 | Logger.error("Failed to delete VM", metadata: ["error": error.localizedDescription])
323 | throw error
324 | }
325 | }
326 |
327 | // MARK: - VM Operations
328 |
329 | @MainActor
330 | public func updateSettings(
331 | name: String,
332 | cpu: Int? = nil,
333 | memory: UInt64? = nil,
334 | diskSize: UInt64? = nil,
335 | display: String? = nil,
336 | storage: String? = nil
337 | ) throws {
338 | let normalizedName = normalizeVMName(name: name)
339 | Logger.info(
340 | "Updating VM settings",
341 | metadata: [
342 | "name": normalizedName,
343 | "location": storage ?? "default",
344 | "cpu": cpu.map { "\($0)" } ?? "unchanged",
345 | "memory": memory.map { "\($0 / 1024 / 1024)MB" } ?? "unchanged",
346 | "disk_size": diskSize.map { "\($0 / 1024 / 1024)MB" } ?? "unchanged",
347 | "display": display ?? "unchanged",
348 | ])
349 | do {
350 | // Find the actual location of the VM
351 | let actualLocation = try self.validateVMExists(
352 | normalizedName, storage: storage)
353 |
354 | let vm = try get(name: normalizedName, storage: actualLocation)
355 |
356 | // Apply settings in order
357 | if let cpu = cpu {
358 | try vm.setCpuCount(cpu)
359 | }
360 | if let memory = memory {
361 | try vm.setMemorySize(memory)
362 | }
363 | if let diskSize = diskSize {
364 | try vm.setDiskSize(diskSize)
365 | }
366 | if let display = display {
367 | try vm.setDisplay(display)
368 | }
369 |
370 | Logger.info("VM settings updated successfully", metadata: ["name": normalizedName])
371 | } catch {
372 | Logger.error(
373 | "Failed to update VM settings", metadata: ["error": error.localizedDescription])
374 | throw error
375 | }
376 | }
377 |
378 | @MainActor
379 | public func stopVM(name: String, storage: String? = nil) async throws {
380 | let normalizedName = normalizeVMName(name: name)
381 | Logger.info("Stopping VM", metadata: ["name": normalizedName])
382 |
383 | do {
384 | // Find the actual location of the VM
385 | let actualLocation = try self.validateVMExists(
386 | normalizedName, storage: storage)
387 |
388 | // Try to get VM from cache first
389 | let vm: VM
390 | if let cachedVM = SharedVM.shared.getVM(name: normalizedName) {
391 | vm = cachedVM
392 | } else {
393 | vm = try get(name: normalizedName, storage: actualLocation)
394 | }
395 |
396 | try await vm.stop()
397 | // Remove VM from cache after stopping
398 | SharedVM.shared.removeVM(name: normalizedName)
399 | Logger.info("VM stopped successfully", metadata: ["name": normalizedName])
400 | } catch {
401 | // Clean up cache even if stop fails
402 | SharedVM.shared.removeVM(name: normalizedName)
403 | Logger.error("Failed to stop VM", metadata: ["error": error.localizedDescription])
404 | throw error
405 | }
406 | }
407 |
408 | @MainActor
409 | public func runVM(
410 | name: String,
411 | noDisplay: Bool = false,
412 | sharedDirectories: [SharedDirectory] = [],
413 | mount: Path? = nil,
414 | registry: String = "ghcr.io",
415 | organization: String = "trycua",
416 | vncPort: Int = 0,
417 | recoveryMode: Bool = false,
418 | storage: String? = nil,
419 | usbMassStoragePaths: [Path]? = nil
420 | ) async throws {
421 | let normalizedName = normalizeVMName(name: name)
422 | Logger.info(
423 | "Running VM",
424 | metadata: [
425 | "name": normalizedName,
426 | "no_display": "\(noDisplay)",
427 | "shared_directories":
428 | "\(sharedDirectories.map( { $0.string } ).joined(separator: ", "))",
429 | "mount": mount?.path ?? "none",
430 | "vnc_port": "\(vncPort)",
431 | "recovery_mode": "\(recoveryMode)",
432 | "storage_param": storage ?? "default", // Log the original param
433 | "usb_storage_devices": "\(usbMassStoragePaths?.count ?? 0)",
434 | ])
435 |
436 | do {
437 | // Check if name is an image ref to auto-pull
438 | let components = normalizedName.split(separator: ":")
439 | if components.count == 2 { // Check if it looks like image:tag
440 | // Attempt to validate if VM exists first, suppressing the error
441 | // This avoids pulling if the VM already exists, even if name looks like an image ref
442 | let vmExists = (try? self.validateVMExists(normalizedName, storage: storage)) != nil
443 | if !vmExists {
444 | Logger.info(
445 | "VM not found, attempting to pull image based on name",
446 | metadata: ["imageRef": normalizedName])
447 | // Use the potentially new VM name derived from the image ref
448 | let potentialVMName = String(components[0])
449 | try await pullImage(
450 | image: normalizedName, // Full image ref
451 | name: potentialVMName, // Name derived from image
452 | registry: registry,
453 | organization: organization,
454 | storage: storage
455 | )
456 | // Important: After pull, the effective name might have changed
457 | // We proceed assuming the user wants to run the VM derived from image name
458 | // normalizedName = potentialVMName // Re-assign normalizedName if pull logic creates it
459 | // Note: Current pullImage doesn't return the final VM name,
460 | // so we assume it matches the name derived from the image.
461 | // This might need refinement if pullImage behaviour changes.
462 | }
463 | }
464 |
465 | // Determine effective storage path or name AND get the VMDirectory
466 | let effectiveStorage: String?
467 | let vmDir: VMDirectory
468 |
469 | if let storagePath = storage, storagePath.contains("/") || storagePath.contains("\\") {
470 | // Storage is a direct path
471 | vmDir = try home.getVMDirectoryFromPath(normalizedName, storagePath: storagePath)
472 | guard vmDir.initialized() else {
473 | if vmDir.exists() {
474 | throw VMError.notInitialized(normalizedName)
475 | } else {
476 | throw VMError.notFound(normalizedName)
477 | }
478 | }
479 | effectiveStorage = storagePath // Use the path string
480 | Logger.info("Using direct storage path", metadata: ["path": storagePath])
481 | } else {
482 | // Storage is nil or a named location - validate and get the actual name
483 | let actualLocationName = try validateVMExists(normalizedName, storage: storage)
484 | vmDir = try home.getVMDirectory(normalizedName, storage: actualLocationName) // Get VMDir for named location
485 | effectiveStorage = actualLocationName // Use the named location string
486 | Logger.info(
487 | "Using named storage location",
488 | metadata: [
489 | "requested": storage ?? "default",
490 | "actual": actualLocationName ?? "default",
491 | ])
492 | }
493 |
494 | // Validate parameters using the located VMDirectory
495 | try validateRunParameters(
496 | vmDir: vmDir, // Pass vmDir
497 | sharedDirectories: sharedDirectories,
498 | mount: mount,
499 | usbMassStoragePaths: usbMassStoragePaths
500 | )
501 |
502 | // Load the VM directly using the located VMDirectory and storage context
503 | let vm = try self.loadVM(vmDir: vmDir, storage: effectiveStorage)
504 |
505 | SharedVM.shared.setVM(name: normalizedName, vm: vm)
506 | try await vm.run(
507 | noDisplay: noDisplay,
508 | sharedDirectories: sharedDirectories,
509 | mount: mount,
510 | vncPort: vncPort,
511 | recoveryMode: recoveryMode,
512 | usbMassStoragePaths: usbMassStoragePaths)
513 | Logger.info("VM started successfully", metadata: ["name": normalizedName])
514 | } catch {
515 | SharedVM.shared.removeVM(name: normalizedName)
516 | Logger.error("Failed to run VM", metadata: ["error": error.localizedDescription])
517 | throw error
518 | }
519 | }
520 |
521 | // MARK: - Image Management
522 |
523 | @MainActor
524 | public func getLatestIPSWURL() async throws -> URL {
525 | Logger.info("Fetching latest supported IPSW URL")
526 |
527 | do {
528 | let imageLoader = DarwinImageLoader()
529 | let url = try await imageLoader.fetchLatestSupportedURL()
530 | Logger.info("Found latest IPSW URL", metadata: ["url": url.absoluteString])
531 | return url
532 | } catch {
533 | Logger.error(
534 | "Failed to fetch IPSW URL", metadata: ["error": error.localizedDescription])
535 | throw error
536 | }
537 | }
538 |
539 | @MainActor
540 | public func pullImage(
541 | image: String,
542 | name: String?,
543 | registry: String,
544 | organization: String,
545 | storage: String? = nil
546 | ) async throws {
547 | do {
548 | // Convert non-sparse image to sparse version if needed
549 | var actualImage = image
550 | var actualName = name
551 |
552 | // Split the image to get name and tag for both sparse and non-sparse cases
553 | let components = image.split(separator: ":")
554 | guard components.count == 2 else {
555 | throw ValidationError("Invalid image format. Expected format: name:tag")
556 | }
557 |
558 | let originalName = String(components[0])
559 | let tag = String(components[1])
560 |
561 | // For consistent VM naming, strip "-sparse" suffix if present when no name provided
562 | let normalizedBaseName: String
563 | if originalName.hasSuffix("-sparse") {
564 | normalizedBaseName = String(originalName.dropLast(7)) // drop "-sparse"
565 | } else {
566 | normalizedBaseName = originalName
567 | }
568 |
569 | // Set default VM name if not provided
570 | if actualName == nil {
571 | actualName = "\(normalizedBaseName)_\(tag)"
572 | }
573 |
574 | // Convert non-sparse image to sparse version if needed
575 | if !image.contains("-sparse") {
576 | // Create sparse version of the image name
577 | actualImage = "\(originalName)-sparse:\(tag)"
578 |
579 | Logger.info(
580 | "Converting to sparse image",
581 | metadata: [
582 | "original": image,
583 | "sparse": actualImage,
584 | "vm_name": actualName ?? "default",
585 | ]
586 | )
587 | }
588 |
589 | let vmName = actualName ?? "default" // Just use actualName as it's already normalized
590 |
591 | Logger.info(
592 | "Pulling image",
593 | metadata: [
594 | "image": actualImage,
595 | "name": vmName,
596 | "registry": registry,
597 | "organization": organization,
598 | "location": storage ?? "default",
599 | ])
600 |
601 | try self.validatePullParameters(
602 | image: actualImage,
603 | name: vmName,
604 | registry: registry,
605 | organization: organization,
606 | storage: storage
607 | )
608 |
609 | let imageContainerRegistry = ImageContainerRegistry(
610 | registry: registry, organization: organization)
611 | let _ = try await imageContainerRegistry.pull(
612 | image: actualImage,
613 | name: vmName,
614 | locationName: storage)
615 |
616 | Logger.info(
617 | "Setting new VM mac address",
618 | metadata: [
619 | "vm_name": vmName,
620 | "location": storage ?? "default",
621 | ])
622 |
623 | // Update MAC address in the cloned VM to ensure uniqueness
624 | let vm = try get(name: vmName, storage: storage)
625 | try vm.setMacAddress(VZMACAddress.randomLocallyAdministered().string)
626 |
627 | Logger.info(
628 | "Image pulled successfully",
629 | metadata: [
630 | "image": actualImage,
631 | "name": vmName,
632 | "registry": registry,
633 | "organization": organization,
634 | "location": storage ?? "default",
635 | ])
636 | } catch {
637 | Logger.error("Failed to pull image", metadata: ["error": error.localizedDescription])
638 | throw error
639 | }
640 | }
641 |
642 | @MainActor
643 | public func pushImage(
644 | name: String,
645 | imageName: String,
646 | tags: [String],
647 | registry: String,
648 | organization: String,
649 | storage: String? = nil,
650 | chunkSizeMb: Int = 512,
651 | verbose: Bool = false,
652 | dryRun: Bool = false,
653 | reassemble: Bool = false
654 | ) async throws {
655 | do {
656 | Logger.info(
657 | "Pushing VM to registry",
658 | metadata: [
659 | "name": name,
660 | "imageName": imageName,
661 | "tags": "\(tags.joined(separator: ", "))",
662 | "registry": registry,
663 | "organization": organization,
664 | "location": storage ?? "default",
665 | "chunk_size": "\(chunkSizeMb)MB",
666 | "dry_run": "\(dryRun)",
667 | "reassemble": "\(reassemble)",
668 | ])
669 |
670 | try validatePushParameters(
671 | name: name,
672 | imageName: imageName,
673 | tags: tags,
674 | registry: registry,
675 | organization: organization
676 | )
677 |
678 | // Find the actual location of the VM
679 | let actualLocation = try self.validateVMExists(name, storage: storage)
680 |
681 | // Get the VM directory
682 | let vmDir = try home.getVMDirectory(name, storage: actualLocation)
683 |
684 | // Use ImageContainerRegistry to push the VM
685 | let imageContainerRegistry = ImageContainerRegistry(
686 | registry: registry, organization: organization)
687 |
688 | try await imageContainerRegistry.push(
689 | vmDirPath: vmDir.dir.path,
690 | imageName: imageName,
691 | tags: tags,
692 | chunkSizeMb: chunkSizeMb,
693 | verbose: verbose,
694 | dryRun: dryRun,
695 | reassemble: reassemble
696 | )
697 |
698 | Logger.info(
699 | "VM pushed successfully",
700 | metadata: [
701 | "name": name,
702 | "imageName": imageName,
703 | "tags": "\(tags.joined(separator: ", "))",
704 | "registry": registry,
705 | "organization": organization,
706 | ])
707 | } catch {
708 | Logger.error("Failed to push VM", metadata: ["error": error.localizedDescription])
709 | throw error
710 | }
711 | }
712 |
713 | @MainActor
714 | public func pruneImages() async throws {
715 | Logger.info("Pruning cached images")
716 |
717 | do {
718 | // Use configured cache directory
719 | let cacheDir = (SettingsManager.shared.getCacheDirectory() as NSString)
720 | .expandingTildeInPath
721 | let ghcrDir = URL(fileURLWithPath: cacheDir).appendingPathComponent("ghcr")
722 |
723 | if FileManager.default.fileExists(atPath: ghcrDir.path) {
724 | try FileManager.default.removeItem(at: ghcrDir)
725 | try FileManager.default.createDirectory(
726 | at: ghcrDir, withIntermediateDirectories: true)
727 | Logger.info("Successfully removed cached images")
728 | } else {
729 | Logger.info("No cached images found")
730 | }
731 | } catch {
732 | Logger.error("Failed to prune images", metadata: ["error": error.localizedDescription])
733 | throw error
734 | }
735 | }
736 |
737 | public struct ImageInfo: Codable {
738 | public let repository: String
739 | public let imageId: String // This will be the shortened manifest ID
740 | }
741 |
742 | public struct ImageList: Codable {
743 | public let local: [ImageInfo]
744 | public let remote: [String] // Keep this for future remote registry support
745 | }
746 |
747 | @MainActor
748 | public func getImages(organization: String = "trycua") async throws -> ImageList {
749 | Logger.info("Listing local images", metadata: ["organization": organization])
750 |
751 | let imageContainerRegistry = ImageContainerRegistry(
752 | registry: "ghcr.io", organization: organization)
753 | let cachedImages = try await imageContainerRegistry.getImages()
754 |
755 | let imageInfos = cachedImages.map { image in
756 | ImageInfo(
757 | repository: image.repository,
758 | imageId: String(image.manifestId.prefix(12))
759 | )
760 | }
761 |
762 | ImagesPrinter.print(images: imageInfos.map { "\($0.repository):\($0.imageId)" })
763 | return ImageList(local: imageInfos, remote: [])
764 | }
765 |
766 | // MARK: - Settings Management
767 |
768 | public func getSettings() -> LumeSettings {
769 | return SettingsManager.shared.getSettings()
770 | }
771 |
772 | public func setHomeDirectory(_ path: String) throws {
773 | // Try to set the home directory in settings
774 | try SettingsManager.shared.setHomeDirectory(path: path)
775 |
776 | // Force recreate home instance to use the new path
777 | try home.validateHomeDirectory()
778 |
779 | Logger.info("Home directory updated", metadata: ["path": path])
780 | }
781 |
782 | // MARK: - VM Location Management
783 |
784 | public func addLocation(name: String, path: String) throws {
785 | Logger.info("Adding VM location", metadata: ["name": name, "path": path])
786 |
787 | try home.addLocation(name: name, path: path)
788 |
789 | Logger.info("VM location added successfully", metadata: ["name": name])
790 | }
791 |
792 | public func removeLocation(name: String) throws {
793 | Logger.info("Removing VM location", metadata: ["name": name])
794 |
795 | try home.removeLocation(name: name)
796 |
797 | Logger.info("VM location removed successfully", metadata: ["name": name])
798 | }
799 |
800 | public func setDefaultLocation(name: String) throws {
801 | Logger.info("Setting default VM location", metadata: ["name": name])
802 |
803 | try home.setDefaultLocation(name: name)
804 |
805 | Logger.info("Default VM location set successfully", metadata: ["name": name])
806 | }
807 |
808 | public func getLocations() -> [VMLocation] {
809 | return home.getLocations()
810 | }
811 |
812 | // MARK: - Cache Directory Management
813 |
814 | public func setCacheDirectory(path: String) throws {
815 | Logger.info("Setting cache directory", metadata: ["path": path])
816 |
817 | try SettingsManager.shared.setCacheDirectory(path: path)
818 |
819 | Logger.info("Cache directory updated", metadata: ["path": path])
820 | }
821 |
822 | public func getCacheDirectory() -> String {
823 | return SettingsManager.shared.getCacheDirectory()
824 | }
825 |
826 | public func isCachingEnabled() -> Bool {
827 | return SettingsManager.shared.isCachingEnabled()
828 | }
829 |
830 | public func setCachingEnabled(_ enabled: Bool) throws {
831 | Logger.info("Setting caching enabled", metadata: ["enabled": "\(enabled)"])
832 |
833 | try SettingsManager.shared.setCachingEnabled(enabled)
834 |
835 | Logger.info("Caching setting updated", metadata: ["enabled": "\(enabled)"])
836 | }
837 |
838 | // MARK: - Private Helper Methods
839 |
840 | /// Normalizes a VM name by replacing colons with underscores
841 | private func normalizeVMName(name: String) -> String {
842 | let components = name.split(separator: ":")
843 | return components.count == 2 ? "\(components[0])_\(components[1])" : name
844 | }
845 |
846 | @MainActor
847 | private func createTempVMConfig(
848 | os: String,
849 | cpuCount: Int,
850 | memorySize: UInt64,
851 | diskSize: UInt64,
852 | display: String
853 | ) async throws -> VM {
854 | let config = try VMConfig(
855 | os: os,
856 | cpuCount: cpuCount,
857 | memorySize: memorySize,
858 | diskSize: diskSize,
859 | macAddress: VZMACAddress.randomLocallyAdministered().string,
860 | display: display
861 | )
862 |
863 | let vmDirContext = VMDirContext(
864 | dir: try home.createTempVMDirectory(),
865 | config: config,
866 | home: home,
867 | storage: nil
868 | )
869 |
870 | let imageLoader = os.lowercased() == "macos" ? imageLoaderFactory.createImageLoader() : nil
871 | return try vmFactory.createVM(vmDirContext: vmDirContext, imageLoader: imageLoader)
872 | }
873 |
874 | @MainActor
875 | private func loadVM(vmDir: VMDirectory, storage: String?) throws -> VM {
876 | // vmDir is now passed directly
877 | guard vmDir.initialized() else {
878 | throw VMError.notInitialized(vmDir.name) // Use name from vmDir
879 | }
880 |
881 | let config: VMConfig = try vmDir.loadConfig()
882 | // Pass the provided storage (which could be a path or named location)
883 | let vmDirContext = VMDirContext(
884 | dir: vmDir, config: config, home: home, storage: storage
885 | )
886 |
887 | let imageLoader =
888 | config.os.lowercased() == "macos" ? imageLoaderFactory.createImageLoader() : nil
889 | return try vmFactory.createVM(vmDirContext: vmDirContext, imageLoader: imageLoader)
890 | }
891 |
892 | // MARK: - Validation Methods
893 |
894 | private func validateCreateParameters(
895 | name: String, os: String, ipsw: String?, storage: String?
896 | ) throws {
897 | if os.lowercased() == "macos" {
898 | guard let ipsw = ipsw else {
899 | throw ValidationError("IPSW path required for macOS VM")
900 | }
901 | if ipsw != "latest" && !FileManager.default.fileExists(atPath: ipsw) {
902 | throw ValidationError("IPSW file not found")
903 | }
904 | } else if os.lowercased() == "linux" {
905 | if ipsw != nil {
906 | throw ValidationError("IPSW path not supported for Linux VM")
907 | }
908 | } else {
909 | throw ValidationError("Unsupported OS type: \(os)")
910 | }
911 |
912 | let vmDir: VMDirectory = try home.getVMDirectory(name, storage: storage)
913 | if vmDir.exists() {
914 | throw VMError.alreadyExists(name)
915 | }
916 | }
917 |
918 | private func validateSharedDirectories(_ directories: [SharedDirectory]) throws {
919 | for dir in directories {
920 | var isDirectory: ObjCBool = false
921 | guard FileManager.default.fileExists(atPath: dir.hostPath, isDirectory: &isDirectory),
922 | isDirectory.boolValue
923 | else {
924 | throw ValidationError(
925 | "Host path does not exist or is not a directory: \(dir.hostPath)")
926 | }
927 | }
928 | }
929 |
930 | public func validateVMExists(_ name: String, storage: String? = nil) throws -> String? {
931 | // If location is specified, only check that location
932 | if let storage = storage {
933 | // Check if storage is a path by looking for directory separator
934 | if storage.contains("/") || storage.contains("\\") {
935 | // Treat as direct path
936 | let vmDir = try home.getVMDirectoryFromPath(name, storagePath: storage)
937 | guard vmDir.initialized() else {
938 | throw VMError.notFound(name)
939 | }
940 | return storage // Return the path as the location identifier
941 | } else {
942 | // Treat as named storage
943 | let vmDir = try home.getVMDirectory(name, storage: storage)
944 | guard vmDir.initialized() else {
945 | throw VMError.notFound(name)
946 | }
947 | return storage
948 | }
949 | }
950 |
951 | // If no location specified, try to find the VM in any location
952 | let allVMs = try home.getAllVMDirectories()
953 | if let foundVM = allVMs.first(where: { $0.directory.name == name }) {
954 | // VM found, return its location
955 | return foundVM.locationName
956 | }
957 |
958 | // VM not found in any location
959 | throw VMError.notFound(name)
960 | }
961 |
962 | private func validateRunParameters(
963 | vmDir: VMDirectory, // Changed signature: accept VMDirectory
964 | sharedDirectories: [SharedDirectory]?,
965 | mount: Path?,
966 | usbMassStoragePaths: [Path]? = nil
967 | ) throws {
968 | // VM existence is confirmed by having vmDir, no need for validateVMExists
969 | if let dirs = sharedDirectories {
970 | try self.validateSharedDirectories(dirs)
971 | }
972 |
973 | // Validate USB mass storage paths
974 | if let usbPaths = usbMassStoragePaths {
975 | for path in usbPaths {
976 | if !FileManager.default.fileExists(atPath: path.path) {
977 | throw ValidationError("USB mass storage image not found: \(path.path)")
978 | }
979 | }
980 |
981 | if #available(macOS 15.0, *) {
982 | // USB mass storage is supported
983 | } else {
984 | Logger.info(
985 | "USB mass storage devices require macOS 15.0 or later. They will be ignored.")
986 | }
987 | }
988 |
989 | // Load config directly from vmDir
990 | let vmConfig = try vmDir.loadConfig()
991 | switch vmConfig.os.lowercased() {
992 | case "macos":
993 | if mount != nil {
994 | throw ValidationError(
995 | "Mounting disk images is not supported for macOS VMs. If you are looking to mount a IPSW, please use the --ipsw option in the create command."
996 | )
997 | }
998 | case "linux":
999 | if let mount = mount, !FileManager.default.fileExists(atPath: mount.path) {
1000 | throw ValidationError("Mount file not found: \(mount.path)")
1001 | }
1002 | default:
1003 | break
1004 | }
1005 | }
1006 |
1007 | private func validatePullParameters(
1008 | image: String,
1009 | name: String,
1010 | registry: String,
1011 | organization: String,
1012 | storage: String? = nil
1013 | ) throws {
1014 | guard !image.isEmpty else {
1015 | throw ValidationError("Image name cannot be empty")
1016 | }
1017 | guard !name.isEmpty else {
1018 | throw ValidationError("VM name cannot be empty")
1019 | }
1020 | guard !registry.isEmpty else {
1021 | throw ValidationError("Registry cannot be empty")
1022 | }
1023 | guard !organization.isEmpty else {
1024 | throw ValidationError("Organization cannot be empty")
1025 | }
1026 |
1027 | // Determine if storage is a path or a named storage location
1028 | let vmDir: VMDirectory
1029 | if let storage = storage, storage.contains("/") || storage.contains("\\") {
1030 | // Create the base directory if it doesn't exist
1031 | if !FileManager.default.fileExists(atPath: storage) {
1032 | Logger.info("Creating VM storage directory", metadata: ["path": storage])
1033 | do {
1034 | try FileManager.default.createDirectory(
1035 | atPath: storage,
1036 | withIntermediateDirectories: true
1037 | )
1038 | } catch {
1039 | throw HomeError.directoryCreationFailed(path: storage)
1040 | }
1041 | }
1042 |
1043 | // Use getVMDirectoryFromPath for direct paths
1044 | vmDir = try home.getVMDirectoryFromPath(name, storagePath: storage)
1045 | } else {
1046 | // Use getVMDirectory for named storage locations
1047 | vmDir = try home.getVMDirectory(name, storage: storage)
1048 | }
1049 |
1050 | if vmDir.exists() {
1051 | throw VMError.alreadyExists(name)
1052 | }
1053 | }
1054 |
1055 | private func validatePushParameters(
1056 | name: String,
1057 | imageName: String,
1058 | tags: [String],
1059 | registry: String,
1060 | organization: String
1061 | ) throws {
1062 | guard !name.isEmpty else {
1063 | throw ValidationError("VM name cannot be empty")
1064 | }
1065 | guard !imageName.isEmpty else {
1066 | throw ValidationError("Image name cannot be empty")
1067 | }
1068 | guard !tags.isEmpty else {
1069 | throw ValidationError("At least one tag must be provided.")
1070 | }
1071 | guard !registry.isEmpty else {
1072 | throw ValidationError("Registry cannot be empty")
1073 | }
1074 | guard !organization.isEmpty else {
1075 | throw ValidationError("Organization cannot be empty")
1076 | }
1077 |
1078 | // Verify VM exists (this will throw if not found)
1079 | _ = try self.validateVMExists(name)
1080 | }
1081 | }
1082 |
```
--------------------------------------------------------------------------------
/libs/python/computer/computer/interface/generic.py:
--------------------------------------------------------------------------------
```python
1 | import asyncio
2 | import json
3 | import time
4 | from typing import Any, Dict, List, Optional, Tuple
5 | from PIL import Image
6 |
7 | import websockets
8 | import aiohttp
9 |
10 | from ..logger import Logger, LogLevel
11 | from .base import BaseComputerInterface
12 | from ..utils import decode_base64_image, encode_base64_image, bytes_to_image, draw_box, resize_image
13 | from .models import Key, KeyType, MouseButton, CommandResult
14 |
15 |
16 | class GenericComputerInterface(BaseComputerInterface):
17 | """Generic interface with common functionality for all supported platforms (Windows, Linux, macOS)."""
18 |
19 | def __init__(self, ip_address: str, username: str = "lume", password: str = "lume", api_key: Optional[str] = None, vm_name: Optional[str] = None, logger_name: str = "computer.interface.generic"):
20 | super().__init__(ip_address, username, password, api_key, vm_name)
21 | self._ws = None
22 | self._reconnect_task = None
23 | self._closed = False
24 | self._last_ping = 0
25 | self._ping_interval = 5 # Send ping every 5 seconds
26 | self._ping_timeout = 120 # Wait 120 seconds for pong response
27 | self._reconnect_delay = 1 # Start with 1 second delay
28 | self._max_reconnect_delay = 30 # Maximum delay between reconnection attempts
29 | self._log_connection_attempts = True # Flag to control connection attempt logging
30 | self._authenticated = False # Track authentication status
31 | self._recv_lock = asyncio.Lock() # Lock to ensure only one recv at a time
32 |
33 | # Set logger name for the interface
34 | self.logger = Logger(logger_name, LogLevel.NORMAL)
35 |
36 | # Optional default delay time between commands (in seconds)
37 | self.delay = 0.0
38 |
39 | async def _handle_delay(self, delay: Optional[float] = None):
40 | """Handle delay between commands using async sleep.
41 |
42 | Args:
43 | delay: Optional delay in seconds. If None, uses self.delay.
44 | """
45 | if delay is not None:
46 | if isinstance(delay, float) or isinstance(delay, int) and delay > 0:
47 | await asyncio.sleep(delay)
48 | elif isinstance(self.delay, float) or isinstance(self.delay, int) and self.delay > 0:
49 | await asyncio.sleep(self.delay)
50 |
51 | @property
52 | def ws_uri(self) -> str:
53 | """Get the WebSocket URI using the current IP address.
54 |
55 | Returns:
56 | WebSocket URI for the Computer API Server
57 | """
58 | protocol = "wss" if self.api_key else "ws"
59 | port = "8443" if self.api_key else "8000"
60 | return f"{protocol}://{self.ip_address}:{port}/ws"
61 |
62 | @property
63 | def rest_uri(self) -> str:
64 | """Get the REST URI using the current IP address.
65 |
66 | Returns:
67 | REST URI for the Computer API Server
68 | """
69 | protocol = "https" if self.api_key else "http"
70 | port = "8443" if self.api_key else "8000"
71 | return f"{protocol}://{self.ip_address}:{port}/cmd"
72 |
73 | # Mouse actions
74 | async def mouse_down(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left", delay: Optional[float] = None) -> None:
75 | await self._send_command("mouse_down", {"x": x, "y": y, "button": button})
76 | await self._handle_delay(delay)
77 |
78 | async def mouse_up(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left", delay: Optional[float] = None) -> None:
79 | await self._send_command("mouse_up", {"x": x, "y": y, "button": button})
80 | await self._handle_delay(delay)
81 |
82 | async def left_click(self, x: Optional[int] = None, y: Optional[int] = None, delay: Optional[float] = None) -> None:
83 | await self._send_command("left_click", {"x": x, "y": y})
84 | await self._handle_delay(delay)
85 |
86 | async def right_click(self, x: Optional[int] = None, y: Optional[int] = None, delay: Optional[float] = None) -> None:
87 | await self._send_command("right_click", {"x": x, "y": y})
88 | await self._handle_delay(delay)
89 |
90 | async def double_click(self, x: Optional[int] = None, y: Optional[int] = None, delay: Optional[float] = None) -> None:
91 | await self._send_command("double_click", {"x": x, "y": y})
92 | await self._handle_delay(delay)
93 |
94 | async def move_cursor(self, x: int, y: int, delay: Optional[float] = None) -> None:
95 | await self._send_command("move_cursor", {"x": x, "y": y})
96 | await self._handle_delay(delay)
97 |
98 | async def drag_to(self, x: int, y: int, button: "MouseButton" = "left", duration: float = 0.5, delay: Optional[float] = None) -> None:
99 | await self._send_command(
100 | "drag_to", {"x": x, "y": y, "button": button, "duration": duration}
101 | )
102 | await self._handle_delay(delay)
103 |
104 | async def drag(self, path: List[Tuple[int, int]], button: "MouseButton" = "left", duration: float = 0.5, delay: Optional[float] = None) -> None:
105 | await self._send_command(
106 | "drag", {"path": path, "button": button, "duration": duration}
107 | )
108 | await self._handle_delay(delay)
109 |
110 | # Keyboard Actions
111 | async def key_down(self, key: "KeyType", delay: Optional[float] = None) -> None:
112 | await self._send_command("key_down", {"key": key})
113 | await self._handle_delay(delay)
114 |
115 | async def key_up(self, key: "KeyType", delay: Optional[float] = None) -> None:
116 | await self._send_command("key_up", {"key": key})
117 | await self._handle_delay(delay)
118 |
119 | async def type_text(self, text: str, delay: Optional[float] = None) -> None:
120 | # Temporary fix for https://github.com/trycua/cua/issues/165
121 | # Check if text contains Unicode characters
122 | if any(ord(char) > 127 for char in text):
123 | # For Unicode text, use clipboard and paste
124 | await self.set_clipboard(text)
125 | await self.hotkey(Key.COMMAND, 'v')
126 | else:
127 | # For ASCII text, use the regular typing method
128 | await self._send_command("type_text", {"text": text})
129 | await self._handle_delay(delay)
130 |
131 | async def press(self, key: "KeyType", delay: Optional[float] = None) -> None:
132 | """Press a single key.
133 |
134 | Args:
135 | key: The key to press. Can be any of:
136 | - A Key enum value (recommended), e.g. Key.PAGE_DOWN
137 | - A direct key value string, e.g. 'pagedown'
138 | - A single character string, e.g. 'a'
139 |
140 | Examples:
141 | ```python
142 | # Using enum (recommended)
143 | await interface.press(Key.PAGE_DOWN)
144 | await interface.press(Key.ENTER)
145 |
146 | # Using direct values
147 | await interface.press('pagedown')
148 | await interface.press('enter')
149 |
150 | # Using single characters
151 | await interface.press('a')
152 | ```
153 |
154 | Raises:
155 | ValueError: If the key type is invalid or the key is not recognized
156 | """
157 | if isinstance(key, Key):
158 | actual_key = key.value
159 | elif isinstance(key, str):
160 | # Try to convert to enum if it matches a known key
161 | key_or_enum = Key.from_string(key)
162 | actual_key = key_or_enum.value if isinstance(key_or_enum, Key) else key_or_enum
163 | else:
164 | raise ValueError(f"Invalid key type: {type(key)}. Must be Key enum or string.")
165 |
166 | await self._send_command("press_key", {"key": actual_key})
167 | await self._handle_delay(delay)
168 |
169 | async def press_key(self, key: "KeyType", delay: Optional[float] = None) -> None:
170 | """DEPRECATED: Use press() instead.
171 |
172 | This method is kept for backward compatibility but will be removed in a future version.
173 | Please use the press() method instead.
174 | """
175 | await self.press(key, delay)
176 |
177 | async def hotkey(self, *keys: "KeyType", delay: Optional[float] = None) -> None:
178 | """Press multiple keys simultaneously.
179 |
180 | Args:
181 | *keys: Multiple keys to press simultaneously. Each key can be any of:
182 | - A Key enum value (recommended), e.g. Key.COMMAND
183 | - A direct key value string, e.g. 'command'
184 | - A single character string, e.g. 'a'
185 |
186 | Examples:
187 | ```python
188 | # Using enums (recommended)
189 | await interface.hotkey(Key.COMMAND, Key.C) # Copy
190 | await interface.hotkey(Key.COMMAND, Key.V) # Paste
191 |
192 | # Using mixed formats
193 | await interface.hotkey(Key.COMMAND, 'a') # Select all
194 | ```
195 |
196 | Raises:
197 | ValueError: If any key type is invalid or not recognized
198 | """
199 | actual_keys = []
200 | for key in keys:
201 | if isinstance(key, Key):
202 | actual_keys.append(key.value)
203 | elif isinstance(key, str):
204 | # Try to convert to enum if it matches a known key
205 | key_or_enum = Key.from_string(key)
206 | actual_keys.append(key_or_enum.value if isinstance(key_or_enum, Key) else key_or_enum)
207 | else:
208 | raise ValueError(f"Invalid key type: {type(key)}. Must be Key enum or string.")
209 |
210 | await self._send_command("hotkey", {"keys": actual_keys})
211 | await self._handle_delay(delay)
212 |
213 | # Scrolling Actions
214 | async def scroll(self, x: int, y: int, delay: Optional[float] = None) -> None:
215 | await self._send_command("scroll", {"x": x, "y": y})
216 | await self._handle_delay(delay)
217 |
218 | async def scroll_down(self, clicks: int = 1, delay: Optional[float] = None) -> None:
219 | await self._send_command("scroll_down", {"clicks": clicks})
220 | await self._handle_delay(delay)
221 |
222 | async def scroll_up(self, clicks: int = 1, delay: Optional[float] = None) -> None:
223 | await self._send_command("scroll_up", {"clicks": clicks})
224 | await self._handle_delay(delay)
225 |
226 | # Screen actions
227 | async def screenshot(
228 | self,
229 | boxes: Optional[List[Tuple[int, int, int, int]]] = None,
230 | box_color: str = "#FF0000",
231 | box_thickness: int = 2,
232 | scale_factor: float = 1.0,
233 | ) -> bytes:
234 | """Take a screenshot with optional box drawing and scaling.
235 |
236 | Args:
237 | boxes: Optional list of (x, y, width, height) tuples defining boxes to draw in screen coordinates
238 | box_color: Color of the boxes in hex format (default: "#FF0000" red)
239 | box_thickness: Thickness of the box borders in pixels (default: 2)
240 | scale_factor: Factor to scale the final image by (default: 1.0)
241 | Use > 1.0 to enlarge, < 1.0 to shrink (e.g., 0.5 for half size, 2.0 for double)
242 |
243 | Returns:
244 | bytes: The screenshot image data, optionally with boxes drawn on it and scaled
245 | """
246 | result = await self._send_command("screenshot")
247 | if not result.get("image_data"):
248 | raise RuntimeError("Failed to take screenshot, no image data received from server")
249 |
250 | screenshot = decode_base64_image(result["image_data"])
251 |
252 | if boxes:
253 | # Get the natural scaling between screen and screenshot
254 | screen_size = await self.get_screen_size()
255 | screenshot_width, screenshot_height = bytes_to_image(screenshot).size
256 | width_scale = screenshot_width / screen_size["width"]
257 | height_scale = screenshot_height / screen_size["height"]
258 |
259 | # Scale box coordinates from screen space to screenshot space
260 | for box in boxes:
261 | scaled_box = (
262 | int(box[0] * width_scale), # x
263 | int(box[1] * height_scale), # y
264 | int(box[2] * width_scale), # width
265 | int(box[3] * height_scale), # height
266 | )
267 | screenshot = draw_box(
268 | screenshot,
269 | x=scaled_box[0],
270 | y=scaled_box[1],
271 | width=scaled_box[2],
272 | height=scaled_box[3],
273 | color=box_color,
274 | thickness=box_thickness,
275 | )
276 |
277 | if scale_factor != 1.0:
278 | screenshot = resize_image(screenshot, scale_factor)
279 |
280 | return screenshot
281 |
282 | async def get_screen_size(self) -> Dict[str, int]:
283 | result = await self._send_command("get_screen_size")
284 | if result["success"] and result["size"]:
285 | return result["size"]
286 | raise RuntimeError("Failed to get screen size")
287 |
288 | async def get_cursor_position(self) -> Dict[str, int]:
289 | result = await self._send_command("get_cursor_position")
290 | if result["success"] and result["position"]:
291 | return result["position"]
292 | raise RuntimeError("Failed to get cursor position")
293 |
294 | # Clipboard Actions
295 | async def copy_to_clipboard(self) -> str:
296 | result = await self._send_command("copy_to_clipboard")
297 | if result["success"] and result["content"]:
298 | return result["content"]
299 | raise RuntimeError("Failed to get clipboard content")
300 |
301 | async def set_clipboard(self, text: str) -> None:
302 | await self._send_command("set_clipboard", {"text": text})
303 |
304 | # File Operations
305 | async def _write_bytes_chunked(self, path: str, content: bytes, append: bool = False, chunk_size: int = 1024 * 1024) -> None:
306 | """Write large files in chunks to avoid memory issues."""
307 | total_size = len(content)
308 | current_offset = 0
309 |
310 | while current_offset < total_size:
311 | chunk_end = min(current_offset + chunk_size, total_size)
312 | chunk_data = content[current_offset:chunk_end]
313 |
314 | # First chunk uses the original append flag, subsequent chunks always append
315 | chunk_append = append if current_offset == 0 else True
316 |
317 | result = await self._send_command("write_bytes", {
318 | "path": path,
319 | "content_b64": encode_base64_image(chunk_data),
320 | "append": chunk_append
321 | })
322 |
323 | if not result.get("success", False):
324 | raise RuntimeError(result.get("error", "Failed to write file chunk"))
325 |
326 | current_offset = chunk_end
327 |
328 | async def write_bytes(self, path: str, content: bytes, append: bool = False) -> None:
329 | # For large files, use chunked writing
330 | if len(content) > 5 * 1024 * 1024: # 5MB threshold
331 | await self._write_bytes_chunked(path, content, append)
332 | return
333 |
334 | result = await self._send_command("write_bytes", {"path": path, "content_b64": encode_base64_image(content), "append": append})
335 | if not result.get("success", False):
336 | raise RuntimeError(result.get("error", "Failed to write file"))
337 |
338 | async def _read_bytes_chunked(self, path: str, offset: int, total_length: int, chunk_size: int = 1024 * 1024) -> bytes:
339 | """Read large files in chunks to avoid memory issues."""
340 | chunks = []
341 | current_offset = offset
342 | remaining = total_length
343 |
344 | while remaining > 0:
345 | read_size = min(chunk_size, remaining)
346 | result = await self._send_command("read_bytes", {
347 | "path": path,
348 | "offset": current_offset,
349 | "length": read_size
350 | })
351 |
352 | if not result.get("success", False):
353 | raise RuntimeError(result.get("error", "Failed to read file chunk"))
354 |
355 | content_b64 = result.get("content_b64", "")
356 | chunk_data = decode_base64_image(content_b64)
357 | chunks.append(chunk_data)
358 |
359 | current_offset += read_size
360 | remaining -= read_size
361 |
362 | return b''.join(chunks)
363 |
364 | async def read_bytes(self, path: str, offset: int = 0, length: Optional[int] = None) -> bytes:
365 | # For large files, use chunked reading
366 | if length is None:
367 | # Get file size first to determine if we need chunking
368 | file_size = await self.get_file_size(path)
369 | # If file is larger than 5MB, read in chunks
370 | if file_size > 5 * 1024 * 1024: # 5MB threshold
371 | return await self._read_bytes_chunked(path, offset, file_size - offset if offset > 0 else file_size)
372 |
373 | result = await self._send_command("read_bytes", {
374 | "path": path,
375 | "offset": offset,
376 | "length": length
377 | })
378 | if not result.get("success", False):
379 | raise RuntimeError(result.get("error", "Failed to read file"))
380 | content_b64 = result.get("content_b64", "")
381 | return decode_base64_image(content_b64)
382 |
383 | async def read_text(self, path: str, encoding: str = 'utf-8') -> str:
384 | """Read text from a file with specified encoding.
385 |
386 | Args:
387 | path: Path to the file to read
388 | encoding: Text encoding to use (default: 'utf-8')
389 |
390 | Returns:
391 | str: The decoded text content of the file
392 | """
393 | content_bytes = await self.read_bytes(path)
394 | return content_bytes.decode(encoding)
395 |
396 | async def write_text(self, path: str, content: str, encoding: str = 'utf-8', append: bool = False) -> None:
397 | """Write text to a file with specified encoding.
398 |
399 | Args:
400 | path: Path to the file to write
401 | content: Text content to write
402 | encoding: Text encoding to use (default: 'utf-8')
403 | append: Whether to append to the file instead of overwriting
404 | """
405 | content_bytes = content.encode(encoding)
406 | await self.write_bytes(path, content_bytes, append)
407 |
408 | async def get_file_size(self, path: str) -> int:
409 | result = await self._send_command("get_file_size", {"path": path})
410 | if not result.get("success", False):
411 | raise RuntimeError(result.get("error", "Failed to get file size"))
412 | return result.get("size", 0)
413 |
414 | async def file_exists(self, path: str) -> bool:
415 | result = await self._send_command("file_exists", {"path": path})
416 | return result.get("exists", False)
417 |
418 | async def directory_exists(self, path: str) -> bool:
419 | result = await self._send_command("directory_exists", {"path": path})
420 | return result.get("exists", False)
421 |
422 | async def create_dir(self, path: str) -> None:
423 | result = await self._send_command("create_dir", {"path": path})
424 | if not result.get("success", False):
425 | raise RuntimeError(result.get("error", "Failed to create directory"))
426 |
427 | async def delete_file(self, path: str) -> None:
428 | result = await self._send_command("delete_file", {"path": path})
429 | if not result.get("success", False):
430 | raise RuntimeError(result.get("error", "Failed to delete file"))
431 |
432 | async def delete_dir(self, path: str) -> None:
433 | result = await self._send_command("delete_dir", {"path": path})
434 | if not result.get("success", False):
435 | raise RuntimeError(result.get("error", "Failed to delete directory"))
436 |
437 | async def list_dir(self, path: str) -> list[str]:
438 | result = await self._send_command("list_dir", {"path": path})
439 | if not result.get("success", False):
440 | raise RuntimeError(result.get("error", "Failed to list directory"))
441 | return result.get("files", [])
442 |
443 | # Command execution
444 | async def run_command(self, command: str) -> CommandResult:
445 | result = await self._send_command("run_command", {"command": command})
446 | if not result.get("success", False):
447 | raise RuntimeError(result.get("error", "Failed to run command"))
448 | return CommandResult(
449 | stdout=result.get("stdout", ""),
450 | stderr=result.get("stderr", ""),
451 | returncode=result.get("return_code", 0)
452 | )
453 |
454 | # Accessibility Actions
455 | async def get_accessibility_tree(self) -> Dict[str, Any]:
456 | """Get the accessibility tree of the current screen."""
457 | result = await self._send_command("get_accessibility_tree")
458 | if not result.get("success", False):
459 | raise RuntimeError(result.get("error", "Failed to get accessibility tree"))
460 | return result
461 |
462 | async def get_active_window_bounds(self) -> Dict[str, int]:
463 | """Get the bounds of the currently active window."""
464 | result = await self._send_command("get_active_window_bounds")
465 | if result["success"] and result["bounds"]:
466 | return result["bounds"]
467 | raise RuntimeError("Failed to get active window bounds")
468 |
469 | async def to_screen_coordinates(self, x: float, y: float) -> tuple[float, float]:
470 | """Convert screenshot coordinates to screen coordinates.
471 |
472 | Args:
473 | x: X coordinate in screenshot space
474 | y: Y coordinate in screenshot space
475 |
476 | Returns:
477 | tuple[float, float]: (x, y) coordinates in screen space
478 | """
479 | screen_size = await self.get_screen_size()
480 | screenshot = await self.screenshot()
481 | screenshot_img = bytes_to_image(screenshot)
482 | screenshot_width, screenshot_height = screenshot_img.size
483 |
484 | # Calculate scaling factors
485 | width_scale = screen_size["width"] / screenshot_width
486 | height_scale = screen_size["height"] / screenshot_height
487 |
488 | # Convert coordinates
489 | screen_x = x * width_scale
490 | screen_y = y * height_scale
491 |
492 | return screen_x, screen_y
493 |
494 | async def to_screenshot_coordinates(self, x: float, y: float) -> tuple[float, float]:
495 | """Convert screen coordinates to screenshot coordinates.
496 |
497 | Args:
498 | x: X coordinate in screen space
499 | y: Y coordinate in screen space
500 |
501 | Returns:
502 | tuple[float, float]: (x, y) coordinates in screenshot space
503 | """
504 | screen_size = await self.get_screen_size()
505 | screenshot = await self.screenshot()
506 | screenshot_img = bytes_to_image(screenshot)
507 | screenshot_width, screenshot_height = screenshot_img.size
508 |
509 | # Calculate scaling factors
510 | width_scale = screenshot_width / screen_size["width"]
511 | height_scale = screenshot_height / screen_size["height"]
512 |
513 | # Convert coordinates
514 | screenshot_x = x * width_scale
515 | screenshot_y = y * height_scale
516 |
517 | return screenshot_x, screenshot_y
518 |
519 | # Websocket Methods
520 | async def _keep_alive(self):
521 | """Keep the WebSocket connection alive with automatic reconnection."""
522 | retry_count = 0
523 | max_log_attempts = 1 # Only log the first attempt at INFO level
524 | log_interval = 500 # Then log every 500th attempt (significantly increased from 30)
525 | last_warning_time = 0
526 | min_warning_interval = 30 # Minimum seconds between connection lost warnings
527 | min_retry_delay = 0.5 # Minimum delay between connection attempts (500ms)
528 |
529 | while not self._closed:
530 | try:
531 | if self._ws is None or (
532 | self._ws and self._ws.state == websockets.protocol.State.CLOSED
533 | ):
534 | try:
535 | retry_count += 1
536 |
537 | # Add a minimum delay between connection attempts to avoid flooding
538 | if retry_count > 1:
539 | await asyncio.sleep(min_retry_delay)
540 |
541 | # Only log the first attempt at INFO level, then every Nth attempt
542 | if retry_count == 1:
543 | self.logger.info(f"Attempting WebSocket connection to {self.ws_uri}")
544 | elif retry_count % log_interval == 0:
545 | self.logger.info(
546 | f"Still attempting WebSocket connection (attempt {retry_count})..."
547 | )
548 | else:
549 | # All other attempts are logged at DEBUG level
550 | self.logger.debug(
551 | f"Attempting WebSocket connection to {self.ws_uri} (attempt {retry_count})"
552 | )
553 |
554 | self._ws = await asyncio.wait_for(
555 | websockets.connect(
556 | self.ws_uri,
557 | max_size=1024 * 1024 * 10, # 10MB limit
558 | max_queue=32,
559 | ping_interval=self._ping_interval,
560 | ping_timeout=self._ping_timeout,
561 | close_timeout=5,
562 | compression=None, # Disable compression to reduce overhead
563 | ),
564 | timeout=120,
565 | )
566 | self.logger.info("WebSocket connection established")
567 |
568 | # If api_key and vm_name are provided, perform authentication handshake
569 | if self.api_key and self.vm_name:
570 | self.logger.info("Performing authentication handshake...")
571 | auth_message = {
572 | "command": "authenticate",
573 | "params": {
574 | "api_key": self.api_key,
575 | "container_name": self.vm_name
576 | }
577 | }
578 | await self._ws.send(json.dumps(auth_message))
579 |
580 | # Wait for authentication response
581 | async with self._recv_lock:
582 | auth_response = await asyncio.wait_for(self._ws.recv(), timeout=10)
583 | auth_result = json.loads(auth_response)
584 |
585 | if not auth_result.get("success"):
586 | error_msg = auth_result.get("error", "Authentication failed")
587 | self.logger.error(f"Authentication failed: {error_msg}")
588 | await self._ws.close()
589 | self._ws = None
590 | raise ConnectionError(f"Authentication failed: {error_msg}")
591 |
592 | self.logger.info("Authentication successful")
593 |
594 | self._reconnect_delay = 1 # Reset reconnect delay on successful connection
595 | self._last_ping = time.time()
596 | retry_count = 0 # Reset retry count on successful connection
597 | except (asyncio.TimeoutError, websockets.exceptions.WebSocketException) as e:
598 | next_retry = self._reconnect_delay
599 |
600 | # Only log the first error at WARNING level, then every Nth attempt
601 | if retry_count == 1:
602 | self.logger.warning(
603 | f"Computer API Server not ready yet. Will retry automatically."
604 | )
605 | elif retry_count % log_interval == 0:
606 | self.logger.warning(
607 | f"Still waiting for Computer API Server (attempt {retry_count})..."
608 | )
609 | else:
610 | # All other errors are logged at DEBUG level
611 | self.logger.debug(f"Connection attempt {retry_count} failed: {e}")
612 |
613 | if self._ws:
614 | try:
615 | await self._ws.close()
616 | except:
617 | pass
618 | self._ws = None
619 |
620 | # Use exponential backoff for connection retries
621 | await asyncio.sleep(self._reconnect_delay)
622 | self._reconnect_delay = min(
623 | self._reconnect_delay * 2, self._max_reconnect_delay
624 | )
625 | continue
626 |
627 | # Regular ping to check connection
628 | if self._ws and self._ws.state == websockets.protocol.State.OPEN:
629 | try:
630 | if time.time() - self._last_ping >= self._ping_interval:
631 | pong_waiter = await self._ws.ping()
632 | await asyncio.wait_for(pong_waiter, timeout=self._ping_timeout)
633 | self._last_ping = time.time()
634 | except Exception as e:
635 | self.logger.debug(f"Ping failed: {e}")
636 | if self._ws:
637 | try:
638 | await self._ws.close()
639 | except:
640 | pass
641 | self._ws = None
642 | continue
643 |
644 | await asyncio.sleep(1)
645 |
646 | except Exception as e:
647 | current_time = time.time()
648 | # Only log connection lost warnings at most once every min_warning_interval seconds
649 | if current_time - last_warning_time >= min_warning_interval:
650 | self.logger.warning(
651 | f"Computer API Server connection lost. Will retry automatically."
652 | )
653 | last_warning_time = current_time
654 | else:
655 | # Log at debug level instead
656 | self.logger.debug(f"Connection lost: {e}")
657 |
658 | if self._ws:
659 | try:
660 | await self._ws.close()
661 | except:
662 | pass
663 | self._ws = None
664 |
665 | async def _ensure_connection(self):
666 | """Ensure WebSocket connection is established."""
667 | if self._reconnect_task is None or self._reconnect_task.done():
668 | self._reconnect_task = asyncio.create_task(self._keep_alive())
669 |
670 | retry_count = 0
671 | max_retries = 5
672 |
673 | while retry_count < max_retries:
674 | try:
675 | if self._ws and self._ws.state == websockets.protocol.State.OPEN:
676 | return
677 | retry_count += 1
678 | await asyncio.sleep(1)
679 | except Exception as e:
680 | # Only log at ERROR level for the last retry attempt
681 | if retry_count == max_retries - 1:
682 | self.logger.error(
683 | f"Persistent connection check error after {retry_count} attempts: {e}"
684 | )
685 | else:
686 | self.logger.debug(f"Connection check error (attempt {retry_count}): {e}")
687 | retry_count += 1
688 | await asyncio.sleep(1)
689 | continue
690 |
691 | raise ConnectionError("Failed to establish WebSocket connection after multiple retries")
692 |
693 | async def _send_command_ws(self, command: str, params: Optional[Dict] = None) -> Dict[str, Any]:
694 | """Send command through WebSocket."""
695 | max_retries = 3
696 | retry_count = 0
697 | last_error = None
698 |
699 | # Acquire lock to ensure only one command is processed at a time
700 | self.logger.debug(f"Acquired lock for command: {command}")
701 | while retry_count < max_retries:
702 | try:
703 | await self._ensure_connection()
704 | if not self._ws:
705 | raise ConnectionError("WebSocket connection is not established")
706 |
707 | message = {"command": command, "params": params or {}}
708 | await self._ws.send(json.dumps(message))
709 | async with self._recv_lock:
710 | response = await asyncio.wait_for(self._ws.recv(), timeout=120)
711 | self.logger.debug(f"Completed command: {command}")
712 | return json.loads(response)
713 | except Exception as e:
714 | last_error = e
715 | retry_count += 1
716 | if retry_count < max_retries:
717 | # Only log at debug level for intermediate retries
718 | self.logger.debug(
719 | f"Command '{command}' failed (attempt {retry_count}/{max_retries}): {e}"
720 | )
721 | await asyncio.sleep(1)
722 | continue
723 | else:
724 | # Only log at error level for the final failure
725 | self.logger.error(
726 | f"Failed to send command '{command}' after {max_retries} retries"
727 | )
728 | self.logger.debug(f"Command failure details: {e}")
729 | raise
730 |
731 | raise last_error if last_error else RuntimeError("Failed to send command")
732 |
733 | async def _send_command_rest(self, command: str, params: Optional[Dict] = None) -> Dict[str, Any]:
734 | """Send command through REST API without retries or connection management."""
735 | try:
736 | # Prepare the request payload
737 | payload = {"command": command, "params": params or {}}
738 |
739 | # Prepare headers
740 | headers = {"Content-Type": "application/json"}
741 | if self.api_key:
742 | headers["X-API-Key"] = self.api_key
743 | if self.vm_name:
744 | headers["X-Container-Name"] = self.vm_name
745 |
746 | # Send the request
747 | async with aiohttp.ClientSession() as session:
748 | async with session.post(
749 | self.rest_uri,
750 | json=payload,
751 | headers=headers
752 | ) as response:
753 | # Get the response text
754 | response_text = await response.text()
755 |
756 | # Trim whitespace
757 | response_text = response_text.strip()
758 |
759 | # Check if it starts with "data: "
760 | if response_text.startswith("data: "):
761 | # Extract everything after "data: "
762 | json_str = response_text[6:] # Remove "data: " prefix
763 | try:
764 | return json.loads(json_str)
765 | except json.JSONDecodeError:
766 | return {
767 | "success": False,
768 | "error": "Server returned malformed response",
769 | "message": response_text
770 | }
771 | else:
772 | # Return error response
773 | return {
774 | "success": False,
775 | "error": "Server returned malformed response",
776 | "message": response_text
777 | }
778 |
779 | except Exception as e:
780 | return {
781 | "success": False,
782 | "error": "Request failed",
783 | "message": str(e)
784 | }
785 |
786 | async def _send_command(self, command: str, params: Optional[Dict] = None) -> Dict[str, Any]:
787 | """Send command using REST API with WebSocket fallback."""
788 | # Try REST API first
789 | result = await self._send_command_rest(command, params)
790 |
791 | # If REST failed with "Request failed", try WebSocket as fallback
792 | if not result.get("success", True) and (result.get("error") == "Request failed" or result.get("error") == "Server returned malformed response"):
793 | self.logger.warning(f"REST API failed for command '{command}', trying WebSocket fallback")
794 | try:
795 | return await self._send_command_ws(command, params)
796 | except Exception as e:
797 | self.logger.error(f"WebSocket fallback also failed: {e}")
798 | # Return the original REST error
799 | return result
800 |
801 | return result
802 |
803 | async def wait_for_ready(self, timeout: int = 60, interval: float = 1.0):
804 | """Wait for Computer API Server to be ready by testing version command."""
805 |
806 | # Check if REST API is available
807 | try:
808 | result = await self._send_command_rest("version", {})
809 | assert result.get("success", True)
810 | except Exception as e:
811 | self.logger.debug(f"REST API failed for command 'version', trying WebSocket fallback: {e}")
812 | try:
813 | await self._wait_for_ready_ws(timeout, interval)
814 | return
815 | except Exception as e:
816 | self.logger.debug(f"WebSocket fallback also failed: {e}")
817 | raise e
818 |
819 | start_time = time.time()
820 | last_error = None
821 | attempt_count = 0
822 | progress_interval = 10 # Log progress every 10 seconds
823 | last_progress_time = start_time
824 |
825 | try:
826 | self.logger.info(
827 | f"Waiting for Computer API Server to be ready (timeout: {timeout}s)..."
828 | )
829 |
830 | # Wait for the server to respond to get_screen_size command
831 | while time.time() - start_time < timeout:
832 | try:
833 | attempt_count += 1
834 | current_time = time.time()
835 |
836 | # Log progress periodically without flooding logs
837 | if current_time - last_progress_time >= progress_interval:
838 | elapsed = current_time - start_time
839 | self.logger.info(
840 | f"Still waiting for Computer API Server... (elapsed: {elapsed:.1f}s, attempts: {attempt_count})"
841 | )
842 | last_progress_time = current_time
843 |
844 | # Test the server with a simple get_screen_size command
845 | result = await self._send_command("get_screen_size")
846 | if result.get("success", False):
847 | elapsed = time.time() - start_time
848 | self.logger.info(
849 | f"Computer API Server is ready (after {elapsed:.1f}s, {attempt_count} attempts)"
850 | )
851 | return # Server is ready
852 | else:
853 | last_error = result.get("error", "Unknown error")
854 | self.logger.debug(f"Initial connection command failed: {last_error}")
855 |
856 | except Exception as e:
857 | last_error = e
858 | self.logger.debug(f"Connection attempt {attempt_count} failed: {e}")
859 |
860 | # Wait before trying again
861 | await asyncio.sleep(interval)
862 |
863 | # If we get here, we've timed out
864 | error_msg = f"Could not connect to {self.ip_address} after {timeout} seconds"
865 | if last_error:
866 | error_msg += f": {str(last_error)}"
867 | self.logger.error(error_msg)
868 | raise TimeoutError(error_msg)
869 |
870 | except Exception as e:
871 | if isinstance(e, TimeoutError):
872 | raise
873 | error_msg = f"Error while waiting for server: {str(e)}"
874 | self.logger.error(error_msg)
875 | raise RuntimeError(error_msg)
876 |
877 | async def _wait_for_ready_ws(self, timeout: int = 60, interval: float = 1.0):
878 | """Wait for WebSocket connection to become available."""
879 | start_time = time.time()
880 | last_error = None
881 | attempt_count = 0
882 | progress_interval = 10 # Log progress every 10 seconds
883 | last_progress_time = start_time
884 |
885 | # Disable detailed logging for connection attempts
886 | self._log_connection_attempts = False
887 |
888 | try:
889 | self.logger.info(
890 | f"Waiting for Computer API Server to be ready (timeout: {timeout}s)..."
891 | )
892 |
893 | # Start the keep-alive task if it's not already running
894 | if self._reconnect_task is None or self._reconnect_task.done():
895 | self._reconnect_task = asyncio.create_task(self._keep_alive())
896 |
897 | # Wait for the connection to be established
898 | while time.time() - start_time < timeout:
899 | try:
900 | attempt_count += 1
901 | current_time = time.time()
902 |
903 | # Log progress periodically without flooding logs
904 | if current_time - last_progress_time >= progress_interval:
905 | elapsed = current_time - start_time
906 | self.logger.info(
907 | f"Still waiting for Computer API Server... (elapsed: {elapsed:.1f}s, attempts: {attempt_count})"
908 | )
909 | last_progress_time = current_time
910 |
911 | # Check if we have a connection
912 | if self._ws and self._ws.state == websockets.protocol.State.OPEN:
913 | # Test the connection with a simple command
914 | try:
915 | await self._send_command_ws("get_screen_size")
916 | elapsed = time.time() - start_time
917 | self.logger.info(
918 | f"Computer API Server is ready (after {elapsed:.1f}s, {attempt_count} attempts)"
919 | )
920 | return # Connection is fully working
921 | except Exception as e:
922 | last_error = e
923 | self.logger.debug(f"Connection test failed: {e}")
924 |
925 | # Wait before trying again
926 | await asyncio.sleep(interval)
927 |
928 | except Exception as e:
929 | last_error = e
930 | self.logger.debug(f"Connection attempt {attempt_count} failed: {e}")
931 | await asyncio.sleep(interval)
932 |
933 | # If we get here, we've timed out
934 | error_msg = f"Could not connect to {self.ip_address} after {timeout} seconds"
935 | if last_error:
936 | error_msg += f": {str(last_error)}"
937 | self.logger.error(error_msg)
938 | raise TimeoutError(error_msg)
939 | finally:
940 | # Reset to default logging behavior
941 | self._log_connection_attempts = False
942 |
943 | def close(self):
944 | """Close WebSocket connection.
945 |
946 | Note: In host computer server mode, we leave the connection open
947 | to allow other clients to connect to the same server. The server
948 | will handle cleaning up idle connections.
949 | """
950 | # Only cancel the reconnect task
951 | if self._reconnect_task:
952 | self._reconnect_task.cancel()
953 |
954 | # Don't set closed flag or close websocket by default
955 | # This allows the server to stay connected for other clients
956 | # self._closed = True
957 | # if self._ws:
958 | # asyncio.create_task(self._ws.close())
959 | # self._ws = None
960 |
961 | def force_close(self):
962 | """Force close the WebSocket connection.
963 |
964 | This method should be called when you want to completely
965 | shut down the connection, not just for regular cleanup.
966 | """
967 | self._closed = True
968 | if self._reconnect_task:
969 | self._reconnect_task.cancel()
970 | if self._ws:
971 | asyncio.create_task(self._ws.close())
972 | self._ws = None
973 |
974 |
```
--------------------------------------------------------------------------------
/libs/python/computer/computer/computer.py:
--------------------------------------------------------------------------------
```python
1 | from typing import Optional, List, Literal, Dict, Any, Union, TYPE_CHECKING, cast
2 | import asyncio
3 | from .models import Computer as ComputerConfig, Display
4 | from .interface.factory import InterfaceFactory
5 | import time
6 | from PIL import Image
7 | import io
8 | import re
9 | from .logger import Logger, LogLevel
10 | import json
11 | import logging
12 | from core.telemetry import is_telemetry_enabled, record_event
13 | import os
14 | from . import helpers
15 |
16 | import platform
17 |
18 | SYSTEM_INFO = {
19 | "os": platform.system().lower(),
20 | "os_version": platform.release(),
21 | "python_version": platform.python_version(),
22 | }
23 |
24 | # Import provider related modules
25 | from .providers.base import VMProviderType
26 | from .providers.factory import VMProviderFactory
27 |
28 | OSType = Literal["macos", "linux", "windows"]
29 |
30 | class Computer:
31 | """Computer is the main class for interacting with the computer."""
32 |
33 | def create_desktop_from_apps(self, apps):
34 | """
35 | Create a virtual desktop from a list of app names, returning a DioramaComputer
36 | that proxies Diorama.Interface but uses diorama_cmds via the computer interface.
37 |
38 | Args:
39 | apps (list[str]): List of application names to include in the desktop.
40 | Returns:
41 | DioramaComputer: A proxy object with the Diorama interface, but using diorama_cmds.
42 | """
43 | assert "app-use" in self.experiments, "App Usage is an experimental feature. Enable it by passing experiments=['app-use'] to Computer()"
44 | from .diorama_computer import DioramaComputer
45 | return DioramaComputer(self, apps)
46 |
47 | def __init__(
48 | self,
49 | display: Union[Display, Dict[str, int], str] = "1024x768",
50 | memory: str = "8GB",
51 | cpu: str = "4",
52 | os_type: OSType = "macos",
53 | name: str = "",
54 | image: Optional[str] = None,
55 | shared_directories: Optional[List[str]] = None,
56 | use_host_computer_server: bool = False,
57 | verbosity: Union[int, LogLevel] = logging.INFO,
58 | telemetry_enabled: bool = True,
59 | provider_type: Union[str, VMProviderType] = VMProviderType.LUME,
60 | port: Optional[int] = 7777,
61 | noVNC_port: Optional[int] = 8006,
62 | host: str = os.environ.get("PYLUME_HOST", "localhost"),
63 | storage: Optional[str] = None,
64 | ephemeral: bool = False,
65 | api_key: Optional[str] = None,
66 | experiments: Optional[List[str]] = None
67 | ):
68 | """Initialize a new Computer instance.
69 |
70 | Args:
71 | display: The display configuration. Can be:
72 | - A Display object
73 | - A dict with 'width' and 'height'
74 | - A string in format "WIDTHxHEIGHT" (e.g. "1920x1080")
75 | Defaults to "1024x768"
76 | memory: The VM memory allocation. Defaults to "8GB"
77 | cpu: The VM CPU allocation. Defaults to "4"
78 | os_type: The operating system type ('macos' or 'linux')
79 | name: The VM name
80 | image: The VM image name
81 | shared_directories: Optional list of directory paths to share with the VM
82 | use_host_computer_server: If True, target localhost instead of starting a VM
83 | verbosity: Logging level (standard Python logging levels: logging.DEBUG, logging.INFO, etc.)
84 | LogLevel enum values are still accepted for backward compatibility
85 | telemetry_enabled: Whether to enable telemetry tracking. Defaults to True.
86 | provider_type: The VM provider type to use (lume, qemu, cloud)
87 | port: Optional port to use for the VM provider server
88 | noVNC_port: Optional port for the noVNC web interface (Lumier provider)
89 | host: Host to use for VM provider connections (e.g. "localhost", "host.docker.internal")
90 | storage: Optional path for persistent VM storage (Lumier provider)
91 | ephemeral: Whether to use ephemeral storage
92 | api_key: Optional API key for cloud providers
93 | experiments: Optional list of experimental features to enable (e.g. ["app-use"])
94 | """
95 |
96 | self.logger = Logger("computer", verbosity)
97 | self.logger.info("Initializing Computer...")
98 |
99 | if not image:
100 | if os_type == "macos":
101 | image = "macos-sequoia-cua:latest"
102 | elif os_type == "linux":
103 | image = "trycua/cua-ubuntu:latest"
104 | image = str(image)
105 |
106 | # Store original parameters
107 | self.image = image
108 | self.port = port
109 | self.noVNC_port = noVNC_port
110 | self.host = host
111 | self.os_type = os_type
112 | self.provider_type = provider_type
113 | self.ephemeral = ephemeral
114 |
115 | self.api_key = api_key
116 | self.experiments = experiments or []
117 |
118 | if "app-use" in self.experiments:
119 | assert self.os_type == "macos", "App use experiment is only supported on macOS"
120 |
121 | # The default is currently to use non-ephemeral storage
122 | if storage and ephemeral and storage != "ephemeral":
123 | raise ValueError("Storage path and ephemeral flag cannot be used together")
124 |
125 | # Windows Sandbox always uses ephemeral storage
126 | if self.provider_type == VMProviderType.WINSANDBOX:
127 | if not ephemeral and storage != None and storage != "ephemeral":
128 | self.logger.warning("Windows Sandbox storage is always ephemeral. Setting ephemeral=True.")
129 | self.ephemeral = True
130 | self.storage = "ephemeral"
131 | else:
132 | self.storage = "ephemeral" if ephemeral else storage
133 |
134 | # For Lumier provider, store the first shared directory path to use
135 | # for VM file sharing
136 | self.shared_path = None
137 | if shared_directories and len(shared_directories) > 0:
138 | self.shared_path = shared_directories[0]
139 | self.logger.info(f"Using first shared directory for VM file sharing: {self.shared_path}")
140 |
141 | # Store telemetry preference
142 | self._telemetry_enabled = telemetry_enabled
143 |
144 | # Set initialization flag
145 | self._initialized = False
146 | self._running = False
147 |
148 | # Configure root logger
149 | self.verbosity = verbosity
150 | self.logger = Logger("computer", verbosity)
151 |
152 | # Configure component loggers with proper hierarchy
153 | self.vm_logger = Logger("computer.vm", verbosity)
154 | self.interface_logger = Logger("computer.interface", verbosity)
155 |
156 | if not use_host_computer_server:
157 | if ":" not in image:
158 | image = f"{image}:latest"
159 |
160 | if not name:
161 | # Normalize the name to be used for the VM
162 | name = image.replace(":", "_")
163 | # Remove any forward slashes
164 | name = name.replace("/", "_")
165 |
166 | # Convert display parameter to Display object
167 | if isinstance(display, str):
168 | # Parse string format "WIDTHxHEIGHT"
169 | match = re.match(r"(\d+)x(\d+)", display)
170 | if not match:
171 | raise ValueError(
172 | "Display string must be in format 'WIDTHxHEIGHT' (e.g. '1024x768')"
173 | )
174 | width, height = map(int, match.groups())
175 | display_config = Display(width=width, height=height)
176 | elif isinstance(display, dict):
177 | display_config = Display(**display)
178 | else:
179 | display_config = display
180 |
181 | self.config = ComputerConfig(
182 | image=image.split(":")[0],
183 | tag=image.split(":")[1],
184 | name=name,
185 | display=display_config,
186 | memory=memory,
187 | cpu=cpu,
188 | )
189 | # Initialize VM provider but don't start it yet - we'll do that in run()
190 | self.config.vm_provider = None # Will be initialized in run()
191 |
192 | # Store shared directories config
193 | self.shared_directories = shared_directories or []
194 |
195 | # Placeholder for VM provider context manager
196 | self._provider_context = None
197 |
198 | # Initialize with proper typing - None at first, will be set in run()
199 | self._interface = None
200 | self.use_host_computer_server = use_host_computer_server
201 |
202 | # Record initialization in telemetry (if enabled)
203 | if telemetry_enabled and is_telemetry_enabled():
204 | record_event("computer_initialized", SYSTEM_INFO)
205 | else:
206 | self.logger.debug("Telemetry disabled - skipping initialization tracking")
207 |
208 | async def __aenter__(self):
209 | """Start the computer."""
210 | await self.run()
211 | return self
212 |
213 | async def __aexit__(self, exc_type, exc_val, exc_tb):
214 | """Stop the computer."""
215 | await self.disconnect()
216 |
217 | def __enter__(self):
218 | """Start the computer."""
219 | # Run the event loop to call the async enter method
220 | loop = asyncio.get_event_loop()
221 | loop.run_until_complete(self.__aenter__())
222 | return self
223 |
224 | def __exit__(self, exc_type, exc_val, exc_tb):
225 | """Stop the computer."""
226 | loop = asyncio.get_event_loop()
227 | loop.run_until_complete(self.__aexit__(exc_type, exc_val, exc_tb))
228 |
229 | async def run(self) -> Optional[str]:
230 | """Initialize the VM and computer interface."""
231 | if TYPE_CHECKING:
232 | from .interface.base import BaseComputerInterface
233 |
234 | # If already initialized, just log and return
235 | if hasattr(self, "_initialized") and self._initialized:
236 | self.logger.info("Computer already initialized, skipping initialization")
237 | return
238 |
239 | self.logger.info("Starting computer...")
240 | start_time = time.time()
241 |
242 | try:
243 | # If using host computer server
244 | if self.use_host_computer_server:
245 | self.logger.info("Using host computer server")
246 | # Set ip_address for host computer server mode
247 | ip_address = "localhost"
248 | # Create the interface with explicit type annotation
249 | from .interface.base import BaseComputerInterface
250 |
251 | self._interface = cast(
252 | BaseComputerInterface,
253 | InterfaceFactory.create_interface_for_os(
254 | os=self.os_type, ip_address=ip_address # type: ignore[arg-type]
255 | ),
256 | )
257 |
258 | self.logger.info("Waiting for host computer server to be ready...")
259 | await self._interface.wait_for_ready()
260 | self.logger.info("Host computer server ready")
261 | else:
262 | # Start or connect to VM
263 | self.logger.info(f"Starting VM: {self.image}")
264 | if not self._provider_context:
265 | try:
266 | provider_type_name = self.provider_type.name if isinstance(self.provider_type, VMProviderType) else self.provider_type
267 | self.logger.verbose(f"Initializing {provider_type_name} provider context...")
268 |
269 | # Explicitly set provider parameters
270 | storage = "ephemeral" if self.ephemeral else self.storage
271 | verbose = self.verbosity >= LogLevel.DEBUG
272 | ephemeral = self.ephemeral
273 | port = self.port if self.port is not None else 7777
274 | host = self.host if self.host else "localhost"
275 | image = self.image
276 | shared_path = self.shared_path
277 | noVNC_port = self.noVNC_port
278 |
279 | # Create VM provider instance with explicit parameters
280 | try:
281 | if self.provider_type == VMProviderType.LUMIER:
282 | self.logger.info(f"Using VM image for Lumier provider: {image}")
283 | if shared_path:
284 | self.logger.info(f"Using shared path for Lumier provider: {shared_path}")
285 | if noVNC_port:
286 | self.logger.info(f"Using noVNC port for Lumier provider: {noVNC_port}")
287 | self.config.vm_provider = VMProviderFactory.create_provider(
288 | self.provider_type,
289 | port=port,
290 | host=host,
291 | storage=storage,
292 | shared_path=shared_path,
293 | image=image,
294 | verbose=verbose,
295 | ephemeral=ephemeral,
296 | noVNC_port=noVNC_port,
297 | )
298 | elif self.provider_type == VMProviderType.LUME:
299 | self.config.vm_provider = VMProviderFactory.create_provider(
300 | self.provider_type,
301 | port=port,
302 | host=host,
303 | storage=storage,
304 | verbose=verbose,
305 | ephemeral=ephemeral,
306 | )
307 | elif self.provider_type == VMProviderType.CLOUD:
308 | self.config.vm_provider = VMProviderFactory.create_provider(
309 | self.provider_type,
310 | api_key=self.api_key,
311 | verbose=verbose,
312 | )
313 | elif self.provider_type == VMProviderType.WINSANDBOX:
314 | self.config.vm_provider = VMProviderFactory.create_provider(
315 | self.provider_type,
316 | port=port,
317 | host=host,
318 | storage=storage,
319 | verbose=verbose,
320 | ephemeral=ephemeral,
321 | noVNC_port=noVNC_port,
322 | )
323 | elif self.provider_type == VMProviderType.DOCKER:
324 | self.config.vm_provider = VMProviderFactory.create_provider(
325 | self.provider_type,
326 | port=port,
327 | host=host,
328 | storage=storage,
329 | shared_path=shared_path,
330 | image=image or "trycua/cua-ubuntu:latest",
331 | verbose=verbose,
332 | ephemeral=ephemeral,
333 | noVNC_port=noVNC_port,
334 | )
335 | else:
336 | raise ValueError(f"Unsupported provider type: {self.provider_type}")
337 | self._provider_context = await self.config.vm_provider.__aenter__()
338 | self.logger.verbose("VM provider context initialized successfully")
339 | except ImportError as ie:
340 | self.logger.error(f"Failed to import provider dependencies: {ie}")
341 | if str(ie).find("lume") >= 0 and str(ie).find("lumier") < 0:
342 | self.logger.error("Please install with: pip install cua-computer[lume]")
343 | elif str(ie).find("lumier") >= 0 or str(ie).find("docker") >= 0:
344 | self.logger.error("Please install with: pip install cua-computer[lumier] and make sure Docker is installed")
345 | elif str(ie).find("cloud") >= 0:
346 | self.logger.error("Please install with: pip install cua-computer[cloud]")
347 | raise
348 | except Exception as e:
349 | self.logger.error(f"Failed to initialize provider context: {e}")
350 | raise RuntimeError(f"Failed to initialize VM provider: {e}")
351 |
352 | # Check if VM exists or create it
353 | is_running = False
354 | try:
355 | if self.config.vm_provider is None:
356 | raise RuntimeError(f"VM provider not initialized for {self.config.name}")
357 |
358 | vm = await self.config.vm_provider.get_vm(self.config.name)
359 | self.logger.verbose(f"Found existing VM: {self.config.name}")
360 | is_running = vm.get("status") == "running"
361 | except Exception as e:
362 | self.logger.error(f"VM not found: {self.config.name}")
363 | self.logger.error(f"Error: {e}")
364 | raise RuntimeError(
365 | f"VM {self.config.name} could not be found or created."
366 | )
367 |
368 | # Start the VM if it's not running
369 | if not is_running:
370 | self.logger.info(f"VM {self.config.name} is not running, starting it...")
371 |
372 | # Convert paths to dictionary format for shared directories
373 | shared_dirs = []
374 | for path in self.shared_directories:
375 | self.logger.verbose(f"Adding shared directory: {path}")
376 | path = os.path.abspath(os.path.expanduser(path))
377 | if os.path.exists(path):
378 | # Add path in format expected by Lume API
379 | shared_dirs.append({
380 | "hostPath": path,
381 | "readOnly": False
382 | })
383 | else:
384 | self.logger.warning(f"Shared directory does not exist: {path}")
385 |
386 | # Prepare run options to pass to the provider
387 | run_opts = {}
388 |
389 | # Add display information if available
390 | if self.config.display is not None:
391 | display_info = {
392 | "width": self.config.display.width,
393 | "height": self.config.display.height,
394 | }
395 |
396 | # Check if scale_factor exists before adding it
397 | if hasattr(self.config.display, "scale_factor"):
398 | display_info["scale_factor"] = self.config.display.scale_factor
399 |
400 | run_opts["display"] = display_info
401 |
402 | # Add shared directories if available
403 | if self.shared_directories:
404 | run_opts["shared_directories"] = shared_dirs.copy()
405 |
406 | # Run the VM with the provider
407 | try:
408 | if self.config.vm_provider is None:
409 | raise RuntimeError(f"VM provider not initialized for {self.config.name}")
410 |
411 | # Use the complete run_opts we prepared earlier
412 | # Handle ephemeral storage for run_vm method too
413 | storage_param = "ephemeral" if self.ephemeral else self.storage
414 |
415 | # Log the image being used
416 | self.logger.info(f"Running VM using image: {self.image}")
417 |
418 | # Call provider.run_vm with explicit image parameter
419 | response = await self.config.vm_provider.run_vm(
420 | image=self.image,
421 | name=self.config.name,
422 | run_opts=run_opts,
423 | storage=storage_param
424 | )
425 | self.logger.info(f"VM run response: {response if response else 'None'}")
426 | except Exception as run_error:
427 | self.logger.error(f"Failed to run VM: {run_error}")
428 | raise RuntimeError(f"Failed to start VM: {run_error}")
429 |
430 | # Wait for VM to be ready with a valid IP address
431 | self.logger.info("Waiting for VM to be ready with a valid IP address...")
432 | try:
433 | if self.provider_type == VMProviderType.LUMIER:
434 | max_retries = 60 # Increased for Lumier VM startup which takes longer
435 | retry_delay = 3 # 3 seconds between retries for Lumier
436 | else:
437 | max_retries = 30 # Default for other providers
438 | retry_delay = 2 # 2 seconds between retries
439 |
440 | self.logger.info(f"Waiting up to {max_retries * retry_delay} seconds for VM to be ready...")
441 | ip = await self.get_ip(max_retries=max_retries, retry_delay=retry_delay)
442 |
443 | # If we get here, we have a valid IP
444 | self.logger.info(f"VM is ready with IP: {ip}")
445 | ip_address = ip
446 | except TimeoutError as timeout_error:
447 | self.logger.error(str(timeout_error))
448 | raise RuntimeError(f"VM startup timed out: {timeout_error}")
449 | except Exception as wait_error:
450 | self.logger.error(f"Error waiting for VM: {wait_error}")
451 | raise RuntimeError(f"VM failed to become ready: {wait_error}")
452 | except Exception as e:
453 | self.logger.error(f"Failed to initialize computer: {e}")
454 | raise RuntimeError(f"Failed to initialize computer: {e}")
455 |
456 | try:
457 | # Verify we have a valid IP before initializing the interface
458 | if not ip_address or ip_address == "unknown" or ip_address == "0.0.0.0":
459 | raise RuntimeError(f"Cannot initialize interface - invalid IP address: {ip_address}")
460 |
461 | # Initialize the interface using the factory with the specified OS
462 | self.logger.info(f"Initializing interface for {self.os_type} at {ip_address}")
463 | from .interface.base import BaseComputerInterface
464 |
465 | # Pass authentication credentials if using cloud provider
466 | if self.provider_type == VMProviderType.CLOUD and self.api_key and self.config.name:
467 | self._interface = cast(
468 | BaseComputerInterface,
469 | InterfaceFactory.create_interface_for_os(
470 | os=self.os_type,
471 | ip_address=ip_address,
472 | api_key=self.api_key,
473 | vm_name=self.config.name
474 | ),
475 | )
476 | else:
477 | self._interface = cast(
478 | BaseComputerInterface,
479 | InterfaceFactory.create_interface_for_os(
480 | os=self.os_type,
481 | ip_address=ip_address
482 | ),
483 | )
484 |
485 | # Wait for the WebSocket interface to be ready
486 | self.logger.info("Connecting to WebSocket interface...")
487 |
488 | try:
489 | # Use a single timeout for the entire connection process
490 | # The VM should already be ready at this point, so we're just establishing the connection
491 | await self._interface.wait_for_ready(timeout=30)
492 | self.logger.info("WebSocket interface connected successfully")
493 | except TimeoutError as e:
494 | self.logger.error(f"Failed to connect to WebSocket interface at {ip_address}")
495 | raise TimeoutError(
496 | f"Could not connect to WebSocket interface at {ip_address}:8000/ws: {str(e)}"
497 | )
498 | # self.logger.warning(
499 | # f"Could not connect to WebSocket interface at {ip_address}:8000/ws: {str(e)}, expect missing functionality"
500 | # )
501 |
502 | # Create an event to keep the VM running in background if needed
503 | if not self.use_host_computer_server:
504 | self._stop_event = asyncio.Event()
505 | self._keep_alive_task = asyncio.create_task(self._stop_event.wait())
506 |
507 | self.logger.info("Computer is ready")
508 |
509 | # Set the initialization flag and clear the initializing flag
510 | self._initialized = True
511 |
512 | # Set this instance as the default computer for remote decorators
513 | helpers.set_default_computer(self)
514 |
515 | self.logger.info("Computer successfully initialized")
516 | except Exception as e:
517 | raise
518 | finally:
519 | # Log initialization time for performance monitoring
520 | duration_ms = (time.time() - start_time) * 1000
521 | self.logger.debug(f"Computer initialization took {duration_ms:.2f}ms")
522 | return
523 |
524 | async def disconnect(self) -> None:
525 | """Disconnect from the computer's WebSocket interface."""
526 | if self._interface:
527 | self._interface.close()
528 |
529 | async def stop(self) -> None:
530 | """Disconnect from the computer's WebSocket interface and stop the computer."""
531 | start_time = time.time()
532 |
533 | try:
534 | self.logger.info("Stopping Computer...")
535 |
536 | # In VM mode, first explicitly stop the VM, then exit the provider context
537 | if not self.use_host_computer_server and self._provider_context and self.config.vm_provider is not None:
538 | try:
539 | self.logger.info(f"Stopping VM {self.config.name}...")
540 | await self.config.vm_provider.stop_vm(
541 | name=self.config.name,
542 | storage=self.storage # Pass storage explicitly for clarity
543 | )
544 | except Exception as e:
545 | self.logger.error(f"Error stopping VM: {e}")
546 |
547 | self.logger.verbose("Closing VM provider context...")
548 | await self.config.vm_provider.__aexit__(None, None, None)
549 | self._provider_context = None
550 |
551 | await self.disconnect()
552 | self.logger.info("Computer stopped")
553 | except Exception as e:
554 | self.logger.debug(f"Error during cleanup: {e}") # Log as debug since this might be expected
555 | finally:
556 | # Log stop time for performance monitoring
557 | duration_ms = (time.time() - start_time) * 1000
558 | self.logger.debug(f"Computer stop process took {duration_ms:.2f}ms")
559 | return
560 |
561 | # @property
562 | async def get_ip(self, max_retries: int = 15, retry_delay: int = 3) -> str:
563 | """Get the IP address of the VM or localhost if using host computer server.
564 |
565 | This method delegates to the provider's get_ip method, which waits indefinitely
566 | until the VM has a valid IP address.
567 |
568 | Args:
569 | max_retries: Unused parameter, kept for backward compatibility
570 | retry_delay: Delay between retries in seconds (default: 2)
571 |
572 | Returns:
573 | IP address of the VM or localhost if using host computer server
574 | """
575 | # For host computer server, always return localhost immediately
576 | if self.use_host_computer_server:
577 | return "127.0.0.1"
578 |
579 | # Get IP from the provider - each provider implements its own waiting logic
580 | if self.config.vm_provider is None:
581 | raise RuntimeError("VM provider is not initialized")
582 |
583 | # Log that we're waiting for the IP
584 | self.logger.info(f"Waiting for VM {self.config.name} to get an IP address...")
585 |
586 | # Call the provider's get_ip method which will wait indefinitely
587 | storage_param = "ephemeral" if self.ephemeral else self.storage
588 |
589 | # Log the image being used
590 | self.logger.info(f"Running VM using image: {self.image}")
591 |
592 | # Call provider.get_ip with explicit image parameter
593 | ip = await self.config.vm_provider.get_ip(
594 | name=self.config.name,
595 | storage=storage_param,
596 | retry_delay=retry_delay
597 | )
598 |
599 | # Log success
600 | self.logger.info(f"VM {self.config.name} has IP address: {ip}")
601 | return ip
602 |
603 |
604 | async def wait_vm_ready(self) -> Optional[Dict[str, Any]]:
605 | """Wait for VM to be ready with an IP address.
606 |
607 | Returns:
608 | VM status information or None if using host computer server.
609 | """
610 | if self.use_host_computer_server:
611 | return None
612 |
613 | timeout = 600 # 10 minutes timeout (increased from 4 minutes)
614 | interval = 2.0 # 2 seconds between checks (increased to reduce API load)
615 | start_time = time.time()
616 | last_status = None
617 | attempts = 0
618 |
619 | self.logger.info(f"Waiting for VM {self.config.name} to be ready (timeout: {timeout}s)...")
620 |
621 | while time.time() - start_time < timeout:
622 | attempts += 1
623 | elapsed = time.time() - start_time
624 |
625 | try:
626 | # Keep polling for VM info
627 | if self.config.vm_provider is None:
628 | self.logger.error("VM provider is not initialized")
629 | vm = None
630 | else:
631 | vm = await self.config.vm_provider.get_vm(self.config.name)
632 |
633 | # Log full VM properties for debugging (every 30 attempts)
634 | if attempts % 30 == 0:
635 | self.logger.info(
636 | f"VM properties at attempt {attempts}: {vars(vm) if vm else 'None'}"
637 | )
638 |
639 | # Get current status for logging
640 | current_status = getattr(vm, "status", None) if vm else None
641 | if current_status != last_status:
642 | self.logger.info(
643 | f"VM status changed to: {current_status} (after {elapsed:.1f}s)"
644 | )
645 | last_status = current_status
646 |
647 | # Check for IP address - ensure it's not None or empty
648 | ip = getattr(vm, "ip_address", None) if vm else None
649 | if ip and ip.strip(): # Check for non-empty string
650 | self.logger.info(
651 | f"VM {self.config.name} got IP address: {ip} (after {elapsed:.1f}s)"
652 | )
653 | return vm
654 |
655 | if attempts % 10 == 0: # Log every 10 attempts to avoid flooding
656 | self.logger.info(
657 | f"Still waiting for VM IP address... (elapsed: {elapsed:.1f}s)"
658 | )
659 | else:
660 | self.logger.debug(
661 | f"Waiting for VM IP address... Current IP: {ip}, Status: {current_status}"
662 | )
663 |
664 | except Exception as e:
665 | self.logger.warning(f"Error checking VM status (attempt {attempts}): {str(e)}")
666 | # If we've been trying for a while and still getting errors, log more details
667 | if elapsed > 60: # After 1 minute of errors, log more details
668 | self.logger.error(f"Persistent error getting VM status: {str(e)}")
669 | self.logger.info("Trying to get VM list for debugging...")
670 | try:
671 | if self.config.vm_provider is not None:
672 | vms = await self.config.vm_provider.list_vms()
673 | self.logger.info(
674 | f"Available VMs: {[getattr(vm, 'name', None) for vm in vms if hasattr(vm, 'name')]}"
675 | )
676 | except Exception as list_error:
677 | self.logger.error(f"Failed to list VMs: {str(list_error)}")
678 |
679 | await asyncio.sleep(interval)
680 |
681 | # If we get here, we've timed out
682 | elapsed = time.time() - start_time
683 | self.logger.error(f"VM {self.config.name} not ready after {elapsed:.1f} seconds")
684 |
685 | # Try to get final VM status for debugging
686 | try:
687 | if self.config.vm_provider is not None:
688 | vm = await self.config.vm_provider.get_vm(self.config.name)
689 | # VM data is returned as a dictionary from the Lumier provider
690 | status = vm.get('status', 'unknown') if vm else "unknown"
691 | ip = vm.get('ip_address') if vm else None
692 | else:
693 | status = "unknown"
694 | ip = None
695 | self.logger.error(f"Final VM status: {status}, IP: {ip}")
696 | except Exception as e:
697 | self.logger.error(f"Failed to get final VM status: {str(e)}")
698 |
699 | raise TimeoutError(
700 | f"VM {self.config.name} not ready after {elapsed:.1f} seconds - IP address not assigned"
701 | )
702 |
703 | async def update(self, cpu: Optional[int] = None, memory: Optional[str] = None):
704 | """Update VM settings."""
705 | self.logger.info(
706 | f"Updating VM settings: CPU={cpu or self.config.cpu}, Memory={memory or self.config.memory}"
707 | )
708 | update_opts = {
709 | "cpu": cpu or int(self.config.cpu),
710 | "memory": memory or self.config.memory
711 | }
712 | if self.config.vm_provider is not None:
713 | await self.config.vm_provider.update_vm(
714 | name=self.config.name,
715 | update_opts=update_opts,
716 | storage=self.storage # Pass storage explicitly for clarity
717 | )
718 | else:
719 | raise RuntimeError("VM provider not initialized")
720 |
721 | def get_screenshot_size(self, screenshot: bytes) -> Dict[str, int]:
722 | """Get the dimensions of a screenshot.
723 |
724 | Args:
725 | screenshot: The screenshot bytes
726 |
727 | Returns:
728 | Dict[str, int]: Dictionary containing 'width' and 'height' of the image
729 | """
730 | image = Image.open(io.BytesIO(screenshot))
731 | width, height = image.size
732 | return {"width": width, "height": height}
733 |
734 | @property
735 | def interface(self):
736 | """Get the computer interface for interacting with the VM.
737 |
738 | Returns:
739 | The computer interface
740 | """
741 | if not hasattr(self, "_interface") or self._interface is None:
742 | error_msg = "Computer interface not initialized. Call run() first."
743 | self.logger.error(error_msg)
744 | self.logger.error(
745 | "Make sure to call await computer.run() before using any interface methods."
746 | )
747 | raise RuntimeError(error_msg)
748 |
749 | return self._interface
750 |
751 | @property
752 | def telemetry_enabled(self) -> bool:
753 | """Check if telemetry is enabled for this computer instance.
754 |
755 | Returns:
756 | bool: True if telemetry is enabled, False otherwise
757 | """
758 | return self._telemetry_enabled
759 |
760 | async def to_screen_coordinates(self, x: float, y: float) -> tuple[float, float]:
761 | """Convert normalized coordinates to screen coordinates.
762 |
763 | Args:
764 | x: X coordinate between 0 and 1
765 | y: Y coordinate between 0 and 1
766 |
767 | Returns:
768 | tuple[float, float]: Screen coordinates (x, y)
769 | """
770 | return await self.interface.to_screen_coordinates(x, y)
771 |
772 | async def to_screenshot_coordinates(self, x: float, y: float) -> tuple[float, float]:
773 | """Convert screen coordinates to screenshot coordinates.
774 |
775 | Args:
776 | x: X coordinate in screen space
777 | y: Y coordinate in screen space
778 |
779 | Returns:
780 | tuple[float, float]: (x, y) coordinates in screenshot space
781 | """
782 | return await self.interface.to_screenshot_coordinates(x, y)
783 |
784 |
785 | # Add virtual environment management functions to computer interface
786 | async def venv_install(self, venv_name: str, requirements: list[str]):
787 | """Install packages in a virtual environment.
788 |
789 | Args:
790 | venv_name: Name of the virtual environment
791 | requirements: List of package requirements to install
792 |
793 | Returns:
794 | Tuple of (stdout, stderr) from the installation command
795 | """
796 | requirements = requirements or []
797 | # Windows vs POSIX handling
798 | if self.os_type == "windows":
799 | # Use %USERPROFILE% for home directory and cmd.exe semantics
800 | venv_path = f"%USERPROFILE%\\.venvs\\{venv_name}"
801 | ensure_dir_cmd = "if not exist \"%USERPROFILE%\\.venvs\" mkdir \"%USERPROFILE%\\.venvs\""
802 | create_cmd = f"if not exist \"{venv_path}\" python -m venv \"{venv_path}\""
803 | requirements_str = " ".join(requirements)
804 | # Activate via activate.bat and install
805 | install_cmd = f"call \"{venv_path}\\Scripts\\activate.bat\" && pip install {requirements_str}" if requirements_str else f"echo No requirements to install"
806 | await self.interface.run_command(ensure_dir_cmd)
807 | await self.interface.run_command(create_cmd)
808 | return await self.interface.run_command(install_cmd)
809 | else:
810 | # POSIX (macOS/Linux)
811 | venv_path = f"$HOME/.venvs/{venv_name}"
812 | create_cmd = f"mkdir -p \"$HOME/.venvs\" && python3 -m venv \"{venv_path}\""
813 | # Check if venv exists, if not create it
814 | check_cmd = f"test -d \"{venv_path}\" || ({create_cmd})"
815 | _ = await self.interface.run_command(check_cmd)
816 | # Install packages
817 | requirements_str = " ".join(requirements)
818 | install_cmd = (
819 | f". \"{venv_path}/bin/activate\" && pip install {requirements_str}"
820 | if requirements_str
821 | else "echo No requirements to install"
822 | )
823 | return await self.interface.run_command(install_cmd)
824 |
825 | async def venv_cmd(self, venv_name: str, command: str):
826 | """Execute a shell command in a virtual environment.
827 |
828 | Args:
829 | venv_name: Name of the virtual environment
830 | command: Shell command to execute in the virtual environment
831 |
832 | Returns:
833 | Tuple of (stdout, stderr) from the command execution
834 | """
835 | if self.os_type == "windows":
836 | # Windows (cmd.exe)
837 | venv_path = f"%USERPROFILE%\\.venvs\\{venv_name}"
838 | # Check existence and signal if missing
839 | check_cmd = f"if not exist \"{venv_path}\" (echo VENV_NOT_FOUND) else (echo VENV_FOUND)"
840 | result = await self.interface.run_command(check_cmd)
841 | if "VENV_NOT_FOUND" in getattr(result, "stdout", ""):
842 | # Auto-create the venv with no requirements
843 | await self.venv_install(venv_name, [])
844 | # Activate and run the command
845 | full_command = f"call \"{venv_path}\\Scripts\\activate.bat\" && {command}"
846 | return await self.interface.run_command(full_command)
847 | else:
848 | # POSIX (macOS/Linux)
849 | venv_path = f"$HOME/.venvs/{venv_name}"
850 | # Check if virtual environment exists
851 | check_cmd = f"test -d \"{venv_path}\""
852 | result = await self.interface.run_command(check_cmd)
853 | if result.stderr or "test:" in result.stdout: # venv doesn't exist
854 | # Auto-create the venv with no requirements
855 | await self.venv_install(venv_name, [])
856 | # Activate virtual environment and run command
857 | full_command = f". \"{venv_path}/bin/activate\" && {command}"
858 | return await self.interface.run_command(full_command)
859 |
860 | async def venv_exec(self, venv_name: str, python_func, *args, **kwargs):
861 | """Execute Python function in a virtual environment using source code extraction.
862 |
863 | Args:
864 | venv_name: Name of the virtual environment
865 | python_func: A callable function to execute
866 | *args: Positional arguments to pass to the function
867 | **kwargs: Keyword arguments to pass to the function
868 |
869 | Returns:
870 | The result of the function execution, or raises any exception that occurred
871 | """
872 | import base64
873 | import inspect
874 | import json
875 | import textwrap
876 |
877 | try:
878 | # Get function source code using inspect.getsource
879 | source = inspect.getsource(python_func)
880 | # Remove common leading whitespace (dedent)
881 | func_source = textwrap.dedent(source).strip()
882 |
883 | # Remove decorators
884 | while func_source.lstrip().startswith("@"):
885 | func_source = func_source.split("\n", 1)[1].strip()
886 |
887 | # Get function name for execution
888 | func_name = python_func.__name__
889 |
890 | # Serialize args and kwargs as JSON (safer than dill for cross-version compatibility)
891 | args_json = json.dumps(args, default=str)
892 | kwargs_json = json.dumps(kwargs, default=str)
893 |
894 | except OSError as e:
895 | raise Exception(f"Cannot retrieve source code for function {python_func.__name__}: {e}")
896 | except Exception as e:
897 | raise Exception(f"Failed to reconstruct function source: {e}")
898 |
899 | # Create Python code that will define and execute the function
900 | python_code = f'''
901 | import json
902 | import traceback
903 |
904 | try:
905 | # Define the function from source
906 | {textwrap.indent(func_source, " ")}
907 |
908 | # Deserialize args and kwargs from JSON
909 | args_json = """{args_json}"""
910 | kwargs_json = """{kwargs_json}"""
911 | args = json.loads(args_json)
912 | kwargs = json.loads(kwargs_json)
913 |
914 | # Execute the function
915 | result = {func_name}(*args, **kwargs)
916 |
917 | # Create success output payload
918 | output_payload = {{
919 | "success": True,
920 | "result": result,
921 | "error": None
922 | }}
923 |
924 | except Exception as e:
925 | # Create error output payload
926 | output_payload = {{
927 | "success": False,
928 | "result": None,
929 | "error": {{
930 | "type": type(e).__name__,
931 | "message": str(e),
932 | "traceback": traceback.format_exc()
933 | }}
934 | }}
935 |
936 | # Serialize the output payload as JSON
937 | import json
938 | output_json = json.dumps(output_payload, default=str)
939 |
940 | # Print the JSON output with markers
941 | print(f"<<<VENV_EXEC_START>>>{{output_json}}<<<VENV_EXEC_END>>>")
942 | '''
943 |
944 | # Encode the Python code in base64 to avoid shell escaping issues
945 | encoded_code = base64.b64encode(python_code.encode('utf-8')).decode('ascii')
946 |
947 | # Execute the Python code in the virtual environment
948 | python_command = f"python -c \"import base64; exec(base64.b64decode('{encoded_code}').decode('utf-8'))\""
949 | result = await self.venv_cmd(venv_name, python_command)
950 |
951 | # Parse the output to extract the payload
952 | start_marker = "<<<VENV_EXEC_START>>>"
953 | end_marker = "<<<VENV_EXEC_END>>>"
954 |
955 | # Print original stdout
956 | print(result.stdout[:result.stdout.find(start_marker)])
957 |
958 | if start_marker in result.stdout and end_marker in result.stdout:
959 | start_idx = result.stdout.find(start_marker) + len(start_marker)
960 | end_idx = result.stdout.find(end_marker)
961 |
962 | if start_idx < end_idx:
963 | output_json = result.stdout[start_idx:end_idx]
964 |
965 | try:
966 | # Decode and deserialize the output payload from JSON
967 | output_payload = json.loads(output_json)
968 | except Exception as e:
969 | raise Exception(f"Failed to decode output payload: {e}")
970 |
971 | if output_payload["success"]:
972 | return output_payload["result"]
973 | else:
974 | # Recreate and raise the original exception
975 | error_info = output_payload["error"]
976 | error_class = eval(error_info["type"])
977 | raise error_class(error_info["message"])
978 | else:
979 | raise Exception("Invalid output format: markers found but no content between them")
980 | else:
981 | # Fallback: return stdout/stderr if no payload markers found
982 | raise Exception(f"No output payload found. stdout: {result.stdout}, stderr: {result.stderr}")
983 |
```