This is page 18 of 21. Use http://codebase.md/trycua/cua?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .all-contributorsrc
├── .cursorignore
├── .devcontainer
│ ├── devcontainer.json
│ ├── post-install.sh
│ └── README.md
├── .dockerignore
├── .gitattributes
├── .github
│ ├── FUNDING.yml
│ ├── scripts
│ │ ├── get_pyproject_version.py
│ │ └── tests
│ │ ├── __init__.py
│ │ ├── README.md
│ │ └── test_get_pyproject_version.py
│ └── workflows
│ ├── ci-lume.yml
│ ├── docker-publish-kasm.yml
│ ├── docker-publish-xfce.yml
│ ├── docker-reusable-publish.yml
│ ├── npm-publish-computer.yml
│ ├── npm-publish-core.yml
│ ├── publish-lume.yml
│ ├── pypi-publish-agent.yml
│ ├── pypi-publish-computer-server.yml
│ ├── pypi-publish-computer.yml
│ ├── pypi-publish-core.yml
│ ├── pypi-publish-mcp-server.yml
│ ├── pypi-publish-pylume.yml
│ ├── pypi-publish-som.yml
│ ├── pypi-reusable-publish.yml
│ └── test-validation-script.yml
├── .gitignore
├── .vscode
│ ├── docs.code-workspace
│ ├── launch.json
│ ├── libs-ts.code-workspace
│ ├── lume.code-workspace
│ ├── lumier.code-workspace
│ ├── py.code-workspace
│ └── settings.json
├── blog
│ ├── app-use.md
│ ├── assets
│ │ ├── composite-agents.png
│ │ ├── docker-ubuntu-support.png
│ │ ├── hack-booth.png
│ │ ├── hack-closing-ceremony.jpg
│ │ ├── hack-cua-ollama-hud.jpeg
│ │ ├── hack-leaderboard.png
│ │ ├── hack-the-north.png
│ │ ├── hack-winners.jpeg
│ │ ├── hack-workshop.jpeg
│ │ ├── hud-agent-evals.png
│ │ └── trajectory-viewer.jpeg
│ ├── bringing-computer-use-to-the-web.md
│ ├── build-your-own-operator-on-macos-1.md
│ ├── build-your-own-operator-on-macos-2.md
│ ├── composite-agents.md
│ ├── cua-hackathon.md
│ ├── hack-the-north.md
│ ├── hud-agent-evals.md
│ ├── human-in-the-loop.md
│ ├── introducing-cua-cloud-containers.md
│ ├── lume-to-containerization.md
│ ├── sandboxed-python-execution.md
│ ├── training-computer-use-models-trajectories-1.md
│ ├── trajectory-viewer.md
│ ├── ubuntu-docker-support.md
│ └── windows-sandbox.md
├── CONTRIBUTING.md
├── Development.md
├── Dockerfile
├── docs
│ ├── .gitignore
│ ├── .prettierrc
│ ├── content
│ │ └── docs
│ │ ├── agent-sdk
│ │ │ ├── agent-loops.mdx
│ │ │ ├── benchmarks
│ │ │ │ ├── index.mdx
│ │ │ │ ├── interactive.mdx
│ │ │ │ ├── introduction.mdx
│ │ │ │ ├── meta.json
│ │ │ │ ├── osworld-verified.mdx
│ │ │ │ ├── screenspot-pro.mdx
│ │ │ │ └── screenspot-v2.mdx
│ │ │ ├── callbacks
│ │ │ │ ├── agent-lifecycle.mdx
│ │ │ │ ├── cost-saving.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── logging.mdx
│ │ │ │ ├── meta.json
│ │ │ │ ├── pii-anonymization.mdx
│ │ │ │ └── trajectories.mdx
│ │ │ ├── chat-history.mdx
│ │ │ ├── custom-computer-handlers.mdx
│ │ │ ├── custom-tools.mdx
│ │ │ ├── customizing-computeragent.mdx
│ │ │ ├── integrations
│ │ │ │ ├── hud.mdx
│ │ │ │ └── meta.json
│ │ │ ├── message-format.mdx
│ │ │ ├── meta.json
│ │ │ ├── migration-guide.mdx
│ │ │ ├── prompt-caching.mdx
│ │ │ ├── supported-agents
│ │ │ │ ├── composed-agents.mdx
│ │ │ │ ├── computer-use-agents.mdx
│ │ │ │ ├── grounding-models.mdx
│ │ │ │ ├── human-in-the-loop.mdx
│ │ │ │ └── meta.json
│ │ │ ├── supported-model-providers
│ │ │ │ ├── index.mdx
│ │ │ │ └── local-models.mdx
│ │ │ └── usage-tracking.mdx
│ │ ├── computer-sdk
│ │ │ ├── cloud-vm-management.mdx
│ │ │ ├── commands.mdx
│ │ │ ├── computer-ui.mdx
│ │ │ ├── computers.mdx
│ │ │ ├── meta.json
│ │ │ └── sandboxed-python.mdx
│ │ ├── index.mdx
│ │ ├── libraries
│ │ │ ├── agent
│ │ │ │ └── index.mdx
│ │ │ ├── computer
│ │ │ │ └── index.mdx
│ │ │ ├── computer-server
│ │ │ │ ├── Commands.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── REST-API.mdx
│ │ │ │ └── WebSocket-API.mdx
│ │ │ ├── core
│ │ │ │ └── index.mdx
│ │ │ ├── lume
│ │ │ │ ├── cli-reference.mdx
│ │ │ │ ├── faq.md
│ │ │ │ ├── http-api.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── installation.mdx
│ │ │ │ ├── meta.json
│ │ │ │ └── prebuilt-images.mdx
│ │ │ ├── lumier
│ │ │ │ ├── building-lumier.mdx
│ │ │ │ ├── docker-compose.mdx
│ │ │ │ ├── docker.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── installation.mdx
│ │ │ │ └── meta.json
│ │ │ ├── mcp-server
│ │ │ │ ├── client-integrations.mdx
│ │ │ │ ├── configuration.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── installation.mdx
│ │ │ │ ├── llm-integrations.mdx
│ │ │ │ ├── meta.json
│ │ │ │ ├── tools.mdx
│ │ │ │ └── usage.mdx
│ │ │ └── som
│ │ │ ├── configuration.mdx
│ │ │ └── index.mdx
│ │ ├── meta.json
│ │ ├── quickstart-cli.mdx
│ │ ├── quickstart-devs.mdx
│ │ └── telemetry.mdx
│ ├── next.config.mjs
│ ├── package-lock.json
│ ├── package.json
│ ├── pnpm-lock.yaml
│ ├── postcss.config.mjs
│ ├── public
│ │ └── img
│ │ ├── agent_gradio_ui.png
│ │ ├── agent.png
│ │ ├── cli.png
│ │ ├── computer.png
│ │ ├── som_box_threshold.png
│ │ └── som_iou_threshold.png
│ ├── README.md
│ ├── source.config.ts
│ ├── src
│ │ ├── app
│ │ │ ├── (home)
│ │ │ │ ├── [[...slug]]
│ │ │ │ │ └── page.tsx
│ │ │ │ └── layout.tsx
│ │ │ ├── api
│ │ │ │ └── search
│ │ │ │ └── route.ts
│ │ │ ├── favicon.ico
│ │ │ ├── global.css
│ │ │ ├── layout.config.tsx
│ │ │ ├── layout.tsx
│ │ │ ├── llms.mdx
│ │ │ │ └── [[...slug]]
│ │ │ │ └── route.ts
│ │ │ └── llms.txt
│ │ │ └── route.ts
│ │ ├── assets
│ │ │ ├── discord-black.svg
│ │ │ ├── discord-white.svg
│ │ │ ├── logo-black.svg
│ │ │ └── logo-white.svg
│ │ ├── components
│ │ │ ├── iou.tsx
│ │ │ └── mermaid.tsx
│ │ ├── lib
│ │ │ ├── llms.ts
│ │ │ └── source.ts
│ │ └── mdx-components.tsx
│ └── tsconfig.json
├── examples
│ ├── agent_examples.py
│ ├── agent_ui_examples.py
│ ├── cloud_api_examples.py
│ ├── computer_examples_windows.py
│ ├── computer_examples.py
│ ├── computer_ui_examples.py
│ ├── computer-example-ts
│ │ ├── .env.example
│ │ ├── .gitignore
│ │ ├── .prettierrc
│ │ ├── package-lock.json
│ │ ├── package.json
│ │ ├── pnpm-lock.yaml
│ │ ├── README.md
│ │ ├── src
│ │ │ ├── helpers.ts
│ │ │ └── index.ts
│ │ └── tsconfig.json
│ ├── docker_examples.py
│ ├── evals
│ │ ├── hud_eval_examples.py
│ │ └── wikipedia_most_linked.txt
│ ├── pylume_examples.py
│ ├── sandboxed_functions_examples.py
│ ├── som_examples.py
│ ├── utils.py
│ └── winsandbox_example.py
├── img
│ ├── agent_gradio_ui.png
│ ├── agent.png
│ ├── cli.png
│ ├── computer.png
│ ├── logo_black.png
│ └── logo_white.png
├── libs
│ ├── kasm
│ │ ├── Dockerfile
│ │ ├── LICENSE
│ │ ├── README.md
│ │ └── src
│ │ └── ubuntu
│ │ └── install
│ │ └── firefox
│ │ ├── custom_startup.sh
│ │ ├── firefox.desktop
│ │ └── install_firefox.sh
│ ├── lume
│ │ ├── .cursorignore
│ │ ├── CONTRIBUTING.md
│ │ ├── Development.md
│ │ ├── img
│ │ │ └── cli.png
│ │ ├── Package.resolved
│ │ ├── Package.swift
│ │ ├── README.md
│ │ ├── resources
│ │ │ └── lume.entitlements
│ │ ├── scripts
│ │ │ ├── build
│ │ │ │ ├── build-debug.sh
│ │ │ │ ├── build-release-notarized.sh
│ │ │ │ └── build-release.sh
│ │ │ └── install.sh
│ │ ├── src
│ │ │ ├── Commands
│ │ │ │ ├── Clone.swift
│ │ │ │ ├── Config.swift
│ │ │ │ ├── Create.swift
│ │ │ │ ├── Delete.swift
│ │ │ │ ├── Get.swift
│ │ │ │ ├── Images.swift
│ │ │ │ ├── IPSW.swift
│ │ │ │ ├── List.swift
│ │ │ │ ├── Logs.swift
│ │ │ │ ├── Options
│ │ │ │ │ └── FormatOption.swift
│ │ │ │ ├── Prune.swift
│ │ │ │ ├── Pull.swift
│ │ │ │ ├── Push.swift
│ │ │ │ ├── Run.swift
│ │ │ │ ├── Serve.swift
│ │ │ │ ├── Set.swift
│ │ │ │ └── Stop.swift
│ │ │ ├── ContainerRegistry
│ │ │ │ ├── ImageContainerRegistry.swift
│ │ │ │ ├── ImageList.swift
│ │ │ │ └── ImagesPrinter.swift
│ │ │ ├── Errors
│ │ │ │ └── Errors.swift
│ │ │ ├── FileSystem
│ │ │ │ ├── Home.swift
│ │ │ │ ├── Settings.swift
│ │ │ │ ├── VMConfig.swift
│ │ │ │ ├── VMDirectory.swift
│ │ │ │ └── VMLocation.swift
│ │ │ ├── LumeController.swift
│ │ │ ├── Main.swift
│ │ │ ├── Server
│ │ │ │ ├── Handlers.swift
│ │ │ │ ├── HTTP.swift
│ │ │ │ ├── Requests.swift
│ │ │ │ ├── Responses.swift
│ │ │ │ └── Server.swift
│ │ │ ├── Utils
│ │ │ │ ├── CommandRegistry.swift
│ │ │ │ ├── CommandUtils.swift
│ │ │ │ ├── Logger.swift
│ │ │ │ ├── NetworkUtils.swift
│ │ │ │ ├── Path.swift
│ │ │ │ ├── ProcessRunner.swift
│ │ │ │ ├── ProgressLogger.swift
│ │ │ │ ├── String.swift
│ │ │ │ └── Utils.swift
│ │ │ ├── Virtualization
│ │ │ │ ├── DarwinImageLoader.swift
│ │ │ │ ├── DHCPLeaseParser.swift
│ │ │ │ ├── ImageLoaderFactory.swift
│ │ │ │ └── VMVirtualizationService.swift
│ │ │ ├── VM
│ │ │ │ ├── DarwinVM.swift
│ │ │ │ ├── LinuxVM.swift
│ │ │ │ ├── VM.swift
│ │ │ │ ├── VMDetails.swift
│ │ │ │ ├── VMDetailsPrinter.swift
│ │ │ │ ├── VMDisplayResolution.swift
│ │ │ │ └── VMFactory.swift
│ │ │ └── VNC
│ │ │ ├── PassphraseGenerator.swift
│ │ │ └── VNCService.swift
│ │ └── tests
│ │ ├── Mocks
│ │ │ ├── MockVM.swift
│ │ │ ├── MockVMVirtualizationService.swift
│ │ │ └── MockVNCService.swift
│ │ ├── VM
│ │ │ └── VMDetailsPrinterTests.swift
│ │ ├── VMTests.swift
│ │ ├── VMVirtualizationServiceTests.swift
│ │ └── VNCServiceTests.swift
│ ├── lumier
│ │ ├── .dockerignore
│ │ ├── Dockerfile
│ │ ├── README.md
│ │ └── src
│ │ ├── bin
│ │ │ └── entry.sh
│ │ ├── config
│ │ │ └── constants.sh
│ │ ├── hooks
│ │ │ └── on-logon.sh
│ │ └── lib
│ │ ├── utils.sh
│ │ └── vm.sh
│ ├── python
│ │ ├── agent
│ │ │ ├── .bumpversion.cfg
│ │ │ ├── agent
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __main__.py
│ │ │ │ ├── adapters
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── huggingfacelocal_adapter.py
│ │ │ │ │ ├── human_adapter.py
│ │ │ │ │ ├── mlxvlm_adapter.py
│ │ │ │ │ └── models
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── generic.py
│ │ │ │ │ ├── internvl.py
│ │ │ │ │ ├── opencua.py
│ │ │ │ │ └── qwen2_5_vl.py
│ │ │ │ ├── agent.py
│ │ │ │ ├── callbacks
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── budget_manager.py
│ │ │ │ │ ├── image_retention.py
│ │ │ │ │ ├── logging.py
│ │ │ │ │ ├── operator_validator.py
│ │ │ │ │ ├── pii_anonymization.py
│ │ │ │ │ ├── prompt_instructions.py
│ │ │ │ │ ├── telemetry.py
│ │ │ │ │ └── trajectory_saver.py
│ │ │ │ ├── cli.py
│ │ │ │ ├── computers
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── cua.py
│ │ │ │ │ └── custom.py
│ │ │ │ ├── decorators.py
│ │ │ │ ├── human_tool
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── __main__.py
│ │ │ │ │ ├── server.py
│ │ │ │ │ └── ui.py
│ │ │ │ ├── integrations
│ │ │ │ │ └── hud
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── agent.py
│ │ │ │ │ └── proxy.py
│ │ │ │ ├── loops
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── anthropic.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── composed_grounded.py
│ │ │ │ │ ├── gemini.py
│ │ │ │ │ ├── glm45v.py
│ │ │ │ │ ├── gta1.py
│ │ │ │ │ ├── holo.py
│ │ │ │ │ ├── internvl.py
│ │ │ │ │ ├── model_types.csv
│ │ │ │ │ ├── moondream3.py
│ │ │ │ │ ├── omniparser.py
│ │ │ │ │ ├── openai.py
│ │ │ │ │ ├── opencua.py
│ │ │ │ │ └── uitars.py
│ │ │ │ ├── proxy
│ │ │ │ │ ├── examples.py
│ │ │ │ │ └── handlers.py
│ │ │ │ ├── responses.py
│ │ │ │ ├── types.py
│ │ │ │ └── ui
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __main__.py
│ │ │ │ └── gradio
│ │ │ │ ├── __init__.py
│ │ │ │ ├── app.py
│ │ │ │ └── ui_components.py
│ │ │ ├── benchmarks
│ │ │ │ ├── .gitignore
│ │ │ │ ├── contrib.md
│ │ │ │ ├── interactive.py
│ │ │ │ ├── models
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ └── gta1.py
│ │ │ │ ├── README.md
│ │ │ │ ├── ss-pro.py
│ │ │ │ ├── ss-v2.py
│ │ │ │ └── utils.py
│ │ │ ├── example.py
│ │ │ ├── pyproject.toml
│ │ │ └── README.md
│ │ ├── computer
│ │ │ ├── .bumpversion.cfg
│ │ │ ├── computer
│ │ │ │ ├── __init__.py
│ │ │ │ ├── computer.py
│ │ │ │ ├── diorama_computer.py
│ │ │ │ ├── helpers.py
│ │ │ │ ├── interface
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── factory.py
│ │ │ │ │ ├── generic.py
│ │ │ │ │ ├── linux.py
│ │ │ │ │ ├── macos.py
│ │ │ │ │ ├── models.py
│ │ │ │ │ └── windows.py
│ │ │ │ ├── logger.py
│ │ │ │ ├── models.py
│ │ │ │ ├── providers
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── cloud
│ │ │ │ │ │ ├── __init__.py
│ │ │ │ │ │ └── provider.py
│ │ │ │ │ ├── docker
│ │ │ │ │ │ ├── __init__.py
│ │ │ │ │ │ └── provider.py
│ │ │ │ │ ├── factory.py
│ │ │ │ │ ├── lume
│ │ │ │ │ │ ├── __init__.py
│ │ │ │ │ │ └── provider.py
│ │ │ │ │ ├── lume_api.py
│ │ │ │ │ ├── lumier
│ │ │ │ │ │ ├── __init__.py
│ │ │ │ │ │ └── provider.py
│ │ │ │ │ ├── types.py
│ │ │ │ │ └── winsandbox
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── provider.py
│ │ │ │ │ └── setup_script.ps1
│ │ │ │ ├── ui
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── __main__.py
│ │ │ │ │ └── gradio
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── app.py
│ │ │ │ └── utils.py
│ │ │ ├── poetry.toml
│ │ │ ├── pyproject.toml
│ │ │ └── README.md
│ │ ├── computer-server
│ │ │ ├── .bumpversion.cfg
│ │ │ ├── computer_server
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __main__.py
│ │ │ │ ├── cli.py
│ │ │ │ ├── diorama
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── diorama_computer.py
│ │ │ │ │ ├── diorama.py
│ │ │ │ │ ├── draw.py
│ │ │ │ │ ├── macos.py
│ │ │ │ │ └── safezone.py
│ │ │ │ ├── handlers
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── factory.py
│ │ │ │ │ ├── generic.py
│ │ │ │ │ ├── linux.py
│ │ │ │ │ ├── macos.py
│ │ │ │ │ └── windows.py
│ │ │ │ ├── main.py
│ │ │ │ ├── server.py
│ │ │ │ └── watchdog.py
│ │ │ ├── examples
│ │ │ │ ├── __init__.py
│ │ │ │ └── usage_example.py
│ │ │ ├── pyproject.toml
│ │ │ ├── README.md
│ │ │ ├── run_server.py
│ │ │ └── test_connection.py
│ │ ├── core
│ │ │ ├── .bumpversion.cfg
│ │ │ ├── core
│ │ │ │ ├── __init__.py
│ │ │ │ └── telemetry
│ │ │ │ ├── __init__.py
│ │ │ │ └── posthog.py
│ │ │ ├── poetry.toml
│ │ │ ├── pyproject.toml
│ │ │ └── README.md
│ │ ├── mcp-server
│ │ │ ├── .bumpversion.cfg
│ │ │ ├── CONCURRENT_SESSIONS.md
│ │ │ ├── mcp_server
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __main__.py
│ │ │ │ ├── server.py
│ │ │ │ └── session_manager.py
│ │ │ ├── pdm.lock
│ │ │ ├── pyproject.toml
│ │ │ ├── README.md
│ │ │ └── scripts
│ │ │ ├── install_mcp_server.sh
│ │ │ └── start_mcp_server.sh
│ │ ├── pylume
│ │ │ ├── __init__.py
│ │ │ ├── .bumpversion.cfg
│ │ │ ├── pylume
│ │ │ │ ├── __init__.py
│ │ │ │ ├── client.py
│ │ │ │ ├── exceptions.py
│ │ │ │ ├── lume
│ │ │ │ ├── models.py
│ │ │ │ ├── pylume.py
│ │ │ │ └── server.py
│ │ │ ├── pyproject.toml
│ │ │ └── README.md
│ │ └── som
│ │ ├── .bumpversion.cfg
│ │ ├── LICENSE
│ │ ├── poetry.toml
│ │ ├── pyproject.toml
│ │ ├── README.md
│ │ ├── som
│ │ │ ├── __init__.py
│ │ │ ├── detect.py
│ │ │ ├── detection.py
│ │ │ ├── models.py
│ │ │ ├── ocr.py
│ │ │ ├── util
│ │ │ │ └── utils.py
│ │ │ └── visualization.py
│ │ └── tests
│ │ └── test_omniparser.py
│ ├── typescript
│ │ ├── .gitignore
│ │ ├── .nvmrc
│ │ ├── agent
│ │ │ ├── examples
│ │ │ │ ├── playground-example.html
│ │ │ │ └── README.md
│ │ │ ├── package.json
│ │ │ ├── README.md
│ │ │ ├── src
│ │ │ │ ├── client.ts
│ │ │ │ ├── index.ts
│ │ │ │ └── types.ts
│ │ │ ├── tests
│ │ │ │ └── client.test.ts
│ │ │ ├── tsconfig.json
│ │ │ ├── tsdown.config.ts
│ │ │ └── vitest.config.ts
│ │ ├── biome.json
│ │ ├── computer
│ │ │ ├── .editorconfig
│ │ │ ├── .gitattributes
│ │ │ ├── .gitignore
│ │ │ ├── LICENSE
│ │ │ ├── package.json
│ │ │ ├── README.md
│ │ │ ├── src
│ │ │ │ ├── computer
│ │ │ │ │ ├── index.ts
│ │ │ │ │ ├── providers
│ │ │ │ │ │ ├── base.ts
│ │ │ │ │ │ ├── cloud.ts
│ │ │ │ │ │ └── index.ts
│ │ │ │ │ └── types.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── interface
│ │ │ │ │ ├── base.ts
│ │ │ │ │ ├── factory.ts
│ │ │ │ │ ├── index.ts
│ │ │ │ │ ├── linux.ts
│ │ │ │ │ ├── macos.ts
│ │ │ │ │ └── windows.ts
│ │ │ │ └── types.ts
│ │ │ ├── tests
│ │ │ │ ├── computer
│ │ │ │ │ └── cloud.test.ts
│ │ │ │ ├── interface
│ │ │ │ │ ├── factory.test.ts
│ │ │ │ │ ├── index.test.ts
│ │ │ │ │ ├── linux.test.ts
│ │ │ │ │ ├── macos.test.ts
│ │ │ │ │ └── windows.test.ts
│ │ │ │ └── setup.ts
│ │ │ ├── tsconfig.json
│ │ │ ├── tsdown.config.ts
│ │ │ └── vitest.config.ts
│ │ ├── core
│ │ │ ├── .editorconfig
│ │ │ ├── .gitattributes
│ │ │ ├── .gitignore
│ │ │ ├── LICENSE
│ │ │ ├── package.json
│ │ │ ├── README.md
│ │ │ ├── src
│ │ │ │ ├── index.ts
│ │ │ │ └── telemetry
│ │ │ │ ├── clients
│ │ │ │ │ ├── index.ts
│ │ │ │ │ └── posthog.ts
│ │ │ │ └── index.ts
│ │ │ ├── tests
│ │ │ │ └── telemetry.test.ts
│ │ │ ├── tsconfig.json
│ │ │ ├── tsdown.config.ts
│ │ │ └── vitest.config.ts
│ │ ├── package.json
│ │ ├── pnpm-lock.yaml
│ │ ├── pnpm-workspace.yaml
│ │ └── README.md
│ └── xfce
│ ├── .dockerignore
│ ├── .gitignore
│ ├── Dockerfile
│ ├── README.md
│ └── src
│ ├── scripts
│ │ ├── resize-display.sh
│ │ ├── start-computer-server.sh
│ │ ├── start-novnc.sh
│ │ ├── start-vnc.sh
│ │ └── xstartup.sh
│ ├── supervisor
│ │ └── supervisord.conf
│ └── xfce-config
│ ├── helpers.rc
│ ├── xfce4-power-manager.xml
│ └── xfce4-session.xml
├── LICENSE.md
├── Makefile
├── notebooks
│ ├── agent_nb.ipynb
│ ├── blog
│ │ ├── build-your-own-operator-on-macos-1.ipynb
│ │ └── build-your-own-operator-on-macos-2.ipynb
│ ├── composite_agents_docker_nb.ipynb
│ ├── computer_nb.ipynb
│ ├── computer_server_nb.ipynb
│ ├── customizing_computeragent.ipynb
│ ├── eval_osworld.ipynb
│ ├── ollama_nb.ipynb
│ ├── pylume_nb.ipynb
│ ├── README.md
│ ├── sota_hackathon_cloud.ipynb
│ └── sota_hackathon.ipynb
├── pdm.lock
├── pyproject.toml
├── pyrightconfig.json
├── README.md
├── samples
│ └── community
│ ├── global-online
│ │ └── README.md
│ └── hack-the-north
│ └── README.md
├── scripts
│ ├── build-uv.sh
│ ├── build.ps1
│ ├── build.sh
│ ├── cleanup.sh
│ ├── playground-docker.sh
│ ├── playground.sh
│ └── run-docker-dev.sh
└── tests
├── pytest.ini
├── shell_cmd.py
├── test_files.py
├── test_mcp_server_session_management.py
├── test_mcp_server_streaming.py
├── test_shell_bash.py
├── test_telemetry.py
├── test_venv.py
└── test_watchdog.py
```
# Files
--------------------------------------------------------------------------------
/libs/python/computer/computer/interface/generic.py:
--------------------------------------------------------------------------------
```python
1 | import asyncio
2 | import json
3 | import time
4 | from typing import Any, Dict, List, Optional, Tuple
5 | from PIL import Image
6 |
7 | import websockets
8 | import aiohttp
9 |
10 | from ..logger import Logger, LogLevel
11 | from .base import BaseComputerInterface
12 | from ..utils import decode_base64_image, encode_base64_image, bytes_to_image, draw_box, resize_image
13 | from .models import Key, KeyType, MouseButton, CommandResult
14 |
15 |
16 | class GenericComputerInterface(BaseComputerInterface):
17 | """Generic interface with common functionality for all supported platforms (Windows, Linux, macOS)."""
18 |
19 | def __init__(self, ip_address: str, username: str = "lume", password: str = "lume", api_key: Optional[str] = None, vm_name: Optional[str] = None, logger_name: str = "computer.interface.generic"):
20 | super().__init__(ip_address, username, password, api_key, vm_name)
21 | self._ws = None
22 | self._reconnect_task = None
23 | self._closed = False
24 | self._last_ping = 0
25 | self._ping_interval = 5 # Send ping every 5 seconds
26 | self._ping_timeout = 120 # Wait 120 seconds for pong response
27 | self._reconnect_delay = 1 # Start with 1 second delay
28 | self._max_reconnect_delay = 30 # Maximum delay between reconnection attempts
29 | self._log_connection_attempts = True # Flag to control connection attempt logging
30 | self._authenticated = False # Track authentication status
31 | self._recv_lock = asyncio.Lock() # Lock to ensure only one recv at a time
32 |
33 | # Set logger name for the interface
34 | self.logger = Logger(logger_name, LogLevel.NORMAL)
35 |
36 | # Optional default delay time between commands (in seconds)
37 | self.delay = 0.0
38 |
39 | async def _handle_delay(self, delay: Optional[float] = None):
40 | """Handle delay between commands using async sleep.
41 |
42 | Args:
43 | delay: Optional delay in seconds. If None, uses self.delay.
44 | """
45 | if delay is not None:
46 | if isinstance(delay, float) or isinstance(delay, int) and delay > 0:
47 | await asyncio.sleep(delay)
48 | elif isinstance(self.delay, float) or isinstance(self.delay, int) and self.delay > 0:
49 | await asyncio.sleep(self.delay)
50 |
51 | @property
52 | def ws_uri(self) -> str:
53 | """Get the WebSocket URI using the current IP address.
54 |
55 | Returns:
56 | WebSocket URI for the Computer API Server
57 | """
58 | protocol = "wss" if self.api_key else "ws"
59 | port = "8443" if self.api_key else "8000"
60 | return f"{protocol}://{self.ip_address}:{port}/ws"
61 |
62 | @property
63 | def rest_uri(self) -> str:
64 | """Get the REST URI using the current IP address.
65 |
66 | Returns:
67 | REST URI for the Computer API Server
68 | """
69 | protocol = "https" if self.api_key else "http"
70 | port = "8443" if self.api_key else "8000"
71 | return f"{protocol}://{self.ip_address}:{port}/cmd"
72 |
73 | # Mouse actions
74 | async def mouse_down(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left", delay: Optional[float] = None) -> None:
75 | await self._send_command("mouse_down", {"x": x, "y": y, "button": button})
76 | await self._handle_delay(delay)
77 |
78 | async def mouse_up(self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left", delay: Optional[float] = None) -> None:
79 | await self._send_command("mouse_up", {"x": x, "y": y, "button": button})
80 | await self._handle_delay(delay)
81 |
82 | async def left_click(self, x: Optional[int] = None, y: Optional[int] = None, delay: Optional[float] = None) -> None:
83 | await self._send_command("left_click", {"x": x, "y": y})
84 | await self._handle_delay(delay)
85 |
86 | async def right_click(self, x: Optional[int] = None, y: Optional[int] = None, delay: Optional[float] = None) -> None:
87 | await self._send_command("right_click", {"x": x, "y": y})
88 | await self._handle_delay(delay)
89 |
90 | async def double_click(self, x: Optional[int] = None, y: Optional[int] = None, delay: Optional[float] = None) -> None:
91 | await self._send_command("double_click", {"x": x, "y": y})
92 | await self._handle_delay(delay)
93 |
94 | async def move_cursor(self, x: int, y: int, delay: Optional[float] = None) -> None:
95 | await self._send_command("move_cursor", {"x": x, "y": y})
96 | await self._handle_delay(delay)
97 |
98 | async def drag_to(self, x: int, y: int, button: "MouseButton" = "left", duration: float = 0.5, delay: Optional[float] = None) -> None:
99 | await self._send_command(
100 | "drag_to", {"x": x, "y": y, "button": button, "duration": duration}
101 | )
102 | await self._handle_delay(delay)
103 |
104 | async def drag(self, path: List[Tuple[int, int]], button: "MouseButton" = "left", duration: float = 0.5, delay: Optional[float] = None) -> None:
105 | await self._send_command(
106 | "drag", {"path": path, "button": button, "duration": duration}
107 | )
108 | await self._handle_delay(delay)
109 |
110 | # Keyboard Actions
111 | async def key_down(self, key: "KeyType", delay: Optional[float] = None) -> None:
112 | await self._send_command("key_down", {"key": key})
113 | await self._handle_delay(delay)
114 |
115 | async def key_up(self, key: "KeyType", delay: Optional[float] = None) -> None:
116 | await self._send_command("key_up", {"key": key})
117 | await self._handle_delay(delay)
118 |
119 | async def type_text(self, text: str, delay: Optional[float] = None) -> None:
120 | await self._send_command("type_text", {"text": text})
121 | await self._handle_delay(delay)
122 |
123 | async def press(self, key: "KeyType", delay: Optional[float] = None) -> None:
124 | """Press a single key.
125 |
126 | Args:
127 | key: The key to press. Can be any of:
128 | - A Key enum value (recommended), e.g. Key.PAGE_DOWN
129 | - A direct key value string, e.g. 'pagedown'
130 | - A single character string, e.g. 'a'
131 |
132 | Examples:
133 | ```python
134 | # Using enum (recommended)
135 | await interface.press(Key.PAGE_DOWN)
136 | await interface.press(Key.ENTER)
137 |
138 | # Using direct values
139 | await interface.press('pagedown')
140 | await interface.press('enter')
141 |
142 | # Using single characters
143 | await interface.press('a')
144 | ```
145 |
146 | Raises:
147 | ValueError: If the key type is invalid or the key is not recognized
148 | """
149 | if isinstance(key, Key):
150 | actual_key = key.value
151 | elif isinstance(key, str):
152 | # Try to convert to enum if it matches a known key
153 | key_or_enum = Key.from_string(key)
154 | actual_key = key_or_enum.value if isinstance(key_or_enum, Key) else key_or_enum
155 | else:
156 | raise ValueError(f"Invalid key type: {type(key)}. Must be Key enum or string.")
157 |
158 | await self._send_command("press_key", {"key": actual_key})
159 | await self._handle_delay(delay)
160 |
161 | async def press_key(self, key: "KeyType", delay: Optional[float] = None) -> None:
162 | """DEPRECATED: Use press() instead.
163 |
164 | This method is kept for backward compatibility but will be removed in a future version.
165 | Please use the press() method instead.
166 | """
167 | await self.press(key, delay)
168 |
169 | async def hotkey(self, *keys: "KeyType", delay: Optional[float] = None) -> None:
170 | """Press multiple keys simultaneously.
171 |
172 | Args:
173 | *keys: Multiple keys to press simultaneously. Each key can be any of:
174 | - A Key enum value (recommended), e.g. Key.COMMAND
175 | - A direct key value string, e.g. 'command'
176 | - A single character string, e.g. 'a'
177 |
178 | Examples:
179 | ```python
180 | # Using enums (recommended)
181 | await interface.hotkey(Key.COMMAND, Key.C) # Copy
182 | await interface.hotkey(Key.COMMAND, Key.V) # Paste
183 |
184 | # Using mixed formats
185 | await interface.hotkey(Key.COMMAND, 'a') # Select all
186 | ```
187 |
188 | Raises:
189 | ValueError: If any key type is invalid or not recognized
190 | """
191 | actual_keys = []
192 | for key in keys:
193 | if isinstance(key, Key):
194 | actual_keys.append(key.value)
195 | elif isinstance(key, str):
196 | # Try to convert to enum if it matches a known key
197 | key_or_enum = Key.from_string(key)
198 | actual_keys.append(key_or_enum.value if isinstance(key_or_enum, Key) else key_or_enum)
199 | else:
200 | raise ValueError(f"Invalid key type: {type(key)}. Must be Key enum or string.")
201 |
202 | await self._send_command("hotkey", {"keys": actual_keys})
203 | await self._handle_delay(delay)
204 |
205 | # Scrolling Actions
206 | async def scroll(self, x: int, y: int, delay: Optional[float] = None) -> None:
207 | await self._send_command("scroll", {"x": x, "y": y})
208 | await self._handle_delay(delay)
209 |
210 | async def scroll_down(self, clicks: int = 1, delay: Optional[float] = None) -> None:
211 | await self._send_command("scroll_down", {"clicks": clicks})
212 | await self._handle_delay(delay)
213 |
214 | async def scroll_up(self, clicks: int = 1, delay: Optional[float] = None) -> None:
215 | await self._send_command("scroll_up", {"clicks": clicks})
216 | await self._handle_delay(delay)
217 |
218 | # Screen actions
219 | async def screenshot(
220 | self,
221 | boxes: Optional[List[Tuple[int, int, int, int]]] = None,
222 | box_color: str = "#FF0000",
223 | box_thickness: int = 2,
224 | scale_factor: float = 1.0,
225 | ) -> bytes:
226 | """Take a screenshot with optional box drawing and scaling.
227 |
228 | Args:
229 | boxes: Optional list of (x, y, width, height) tuples defining boxes to draw in screen coordinates
230 | box_color: Color of the boxes in hex format (default: "#FF0000" red)
231 | box_thickness: Thickness of the box borders in pixels (default: 2)
232 | scale_factor: Factor to scale the final image by (default: 1.0)
233 | Use > 1.0 to enlarge, < 1.0 to shrink (e.g., 0.5 for half size, 2.0 for double)
234 |
235 | Returns:
236 | bytes: The screenshot image data, optionally with boxes drawn on it and scaled
237 | """
238 | result = await self._send_command("screenshot")
239 | if not result.get("image_data"):
240 | raise RuntimeError("Failed to take screenshot, no image data received from server")
241 |
242 | screenshot = decode_base64_image(result["image_data"])
243 |
244 | if boxes:
245 | # Get the natural scaling between screen and screenshot
246 | screen_size = await self.get_screen_size()
247 | screenshot_width, screenshot_height = bytes_to_image(screenshot).size
248 | width_scale = screenshot_width / screen_size["width"]
249 | height_scale = screenshot_height / screen_size["height"]
250 |
251 | # Scale box coordinates from screen space to screenshot space
252 | for box in boxes:
253 | scaled_box = (
254 | int(box[0] * width_scale), # x
255 | int(box[1] * height_scale), # y
256 | int(box[2] * width_scale), # width
257 | int(box[3] * height_scale), # height
258 | )
259 | screenshot = draw_box(
260 | screenshot,
261 | x=scaled_box[0],
262 | y=scaled_box[1],
263 | width=scaled_box[2],
264 | height=scaled_box[3],
265 | color=box_color,
266 | thickness=box_thickness,
267 | )
268 |
269 | if scale_factor != 1.0:
270 | screenshot = resize_image(screenshot, scale_factor)
271 |
272 | return screenshot
273 |
274 | async def get_screen_size(self) -> Dict[str, int]:
275 | result = await self._send_command("get_screen_size")
276 | if result["success"] and result["size"]:
277 | return result["size"]
278 | raise RuntimeError("Failed to get screen size")
279 |
280 | async def get_cursor_position(self) -> Dict[str, int]:
281 | result = await self._send_command("get_cursor_position")
282 | if result["success"] and result["position"]:
283 | return result["position"]
284 | raise RuntimeError("Failed to get cursor position")
285 |
286 | # Clipboard Actions
287 | async def copy_to_clipboard(self) -> str:
288 | result = await self._send_command("copy_to_clipboard")
289 | if result["success"] and result["content"]:
290 | return result["content"]
291 | raise RuntimeError("Failed to get clipboard content")
292 |
293 | async def set_clipboard(self, text: str) -> None:
294 | await self._send_command("set_clipboard", {"text": text})
295 |
296 | # File Operations
297 | async def _write_bytes_chunked(self, path: str, content: bytes, append: bool = False, chunk_size: int = 1024 * 1024) -> None:
298 | """Write large files in chunks to avoid memory issues."""
299 | total_size = len(content)
300 | current_offset = 0
301 |
302 | while current_offset < total_size:
303 | chunk_end = min(current_offset + chunk_size, total_size)
304 | chunk_data = content[current_offset:chunk_end]
305 |
306 | # First chunk uses the original append flag, subsequent chunks always append
307 | chunk_append = append if current_offset == 0 else True
308 |
309 | result = await self._send_command("write_bytes", {
310 | "path": path,
311 | "content_b64": encode_base64_image(chunk_data),
312 | "append": chunk_append
313 | })
314 |
315 | if not result.get("success", False):
316 | raise RuntimeError(result.get("error", "Failed to write file chunk"))
317 |
318 | current_offset = chunk_end
319 |
320 | async def write_bytes(self, path: str, content: bytes, append: bool = False) -> None:
321 | # For large files, use chunked writing
322 | if len(content) > 5 * 1024 * 1024: # 5MB threshold
323 | await self._write_bytes_chunked(path, content, append)
324 | return
325 |
326 | result = await self._send_command("write_bytes", {"path": path, "content_b64": encode_base64_image(content), "append": append})
327 | if not result.get("success", False):
328 | raise RuntimeError(result.get("error", "Failed to write file"))
329 |
330 | async def _read_bytes_chunked(self, path: str, offset: int, total_length: int, chunk_size: int = 1024 * 1024) -> bytes:
331 | """Read large files in chunks to avoid memory issues."""
332 | chunks = []
333 | current_offset = offset
334 | remaining = total_length
335 |
336 | while remaining > 0:
337 | read_size = min(chunk_size, remaining)
338 | result = await self._send_command("read_bytes", {
339 | "path": path,
340 | "offset": current_offset,
341 | "length": read_size
342 | })
343 |
344 | if not result.get("success", False):
345 | raise RuntimeError(result.get("error", "Failed to read file chunk"))
346 |
347 | content_b64 = result.get("content_b64", "")
348 | chunk_data = decode_base64_image(content_b64)
349 | chunks.append(chunk_data)
350 |
351 | current_offset += read_size
352 | remaining -= read_size
353 |
354 | return b''.join(chunks)
355 |
356 | async def read_bytes(self, path: str, offset: int = 0, length: Optional[int] = None) -> bytes:
357 | # For large files, use chunked reading
358 | if length is None:
359 | # Get file size first to determine if we need chunking
360 | file_size = await self.get_file_size(path)
361 | # If file is larger than 5MB, read in chunks
362 | if file_size > 5 * 1024 * 1024: # 5MB threshold
363 | return await self._read_bytes_chunked(path, offset, file_size - offset if offset > 0 else file_size)
364 |
365 | result = await self._send_command("read_bytes", {
366 | "path": path,
367 | "offset": offset,
368 | "length": length
369 | })
370 | if not result.get("success", False):
371 | raise RuntimeError(result.get("error", "Failed to read file"))
372 | content_b64 = result.get("content_b64", "")
373 | return decode_base64_image(content_b64)
374 |
375 | async def read_text(self, path: str, encoding: str = 'utf-8') -> str:
376 | """Read text from a file with specified encoding.
377 |
378 | Args:
379 | path: Path to the file to read
380 | encoding: Text encoding to use (default: 'utf-8')
381 |
382 | Returns:
383 | str: The decoded text content of the file
384 | """
385 | content_bytes = await self.read_bytes(path)
386 | return content_bytes.decode(encoding)
387 |
388 | async def write_text(self, path: str, content: str, encoding: str = 'utf-8', append: bool = False) -> None:
389 | """Write text to a file with specified encoding.
390 |
391 | Args:
392 | path: Path to the file to write
393 | content: Text content to write
394 | encoding: Text encoding to use (default: 'utf-8')
395 | append: Whether to append to the file instead of overwriting
396 | """
397 | content_bytes = content.encode(encoding)
398 | await self.write_bytes(path, content_bytes, append)
399 |
400 | async def get_file_size(self, path: str) -> int:
401 | result = await self._send_command("get_file_size", {"path": path})
402 | if not result.get("success", False):
403 | raise RuntimeError(result.get("error", "Failed to get file size"))
404 | return result.get("size", 0)
405 |
406 | async def file_exists(self, path: str) -> bool:
407 | result = await self._send_command("file_exists", {"path": path})
408 | return result.get("exists", False)
409 |
410 | async def directory_exists(self, path: str) -> bool:
411 | result = await self._send_command("directory_exists", {"path": path})
412 | return result.get("exists", False)
413 |
414 | async def create_dir(self, path: str) -> None:
415 | result = await self._send_command("create_dir", {"path": path})
416 | if not result.get("success", False):
417 | raise RuntimeError(result.get("error", "Failed to create directory"))
418 |
419 | async def delete_file(self, path: str) -> None:
420 | result = await self._send_command("delete_file", {"path": path})
421 | if not result.get("success", False):
422 | raise RuntimeError(result.get("error", "Failed to delete file"))
423 |
424 | async def delete_dir(self, path: str) -> None:
425 | result = await self._send_command("delete_dir", {"path": path})
426 | if not result.get("success", False):
427 | raise RuntimeError(result.get("error", "Failed to delete directory"))
428 |
429 | async def list_dir(self, path: str) -> list[str]:
430 | result = await self._send_command("list_dir", {"path": path})
431 | if not result.get("success", False):
432 | raise RuntimeError(result.get("error", "Failed to list directory"))
433 | return result.get("files", [])
434 |
435 | # Command execution
436 | async def run_command(self, command: str) -> CommandResult:
437 | result = await self._send_command("run_command", {"command": command})
438 | if not result.get("success", False):
439 | raise RuntimeError(result.get("error", "Failed to run command"))
440 | return CommandResult(
441 | stdout=result.get("stdout", ""),
442 | stderr=result.get("stderr", ""),
443 | returncode=result.get("return_code", 0)
444 | )
445 |
446 | # Accessibility Actions
447 | async def get_accessibility_tree(self) -> Dict[str, Any]:
448 | """Get the accessibility tree of the current screen."""
449 | result = await self._send_command("get_accessibility_tree")
450 | if not result.get("success", False):
451 | raise RuntimeError(result.get("error", "Failed to get accessibility tree"))
452 | return result
453 |
454 | async def get_active_window_bounds(self) -> Dict[str, int]:
455 | """Get the bounds of the currently active window."""
456 | result = await self._send_command("get_active_window_bounds")
457 | if result["success"] and result["bounds"]:
458 | return result["bounds"]
459 | raise RuntimeError("Failed to get active window bounds")
460 |
461 | async def to_screen_coordinates(self, x: float, y: float) -> tuple[float, float]:
462 | """Convert screenshot coordinates to screen coordinates.
463 |
464 | Args:
465 | x: X coordinate in screenshot space
466 | y: Y coordinate in screenshot space
467 |
468 | Returns:
469 | tuple[float, float]: (x, y) coordinates in screen space
470 | """
471 | screen_size = await self.get_screen_size()
472 | screenshot = await self.screenshot()
473 | screenshot_img = bytes_to_image(screenshot)
474 | screenshot_width, screenshot_height = screenshot_img.size
475 |
476 | # Calculate scaling factors
477 | width_scale = screen_size["width"] / screenshot_width
478 | height_scale = screen_size["height"] / screenshot_height
479 |
480 | # Convert coordinates
481 | screen_x = x * width_scale
482 | screen_y = y * height_scale
483 |
484 | return screen_x, screen_y
485 |
486 | async def to_screenshot_coordinates(self, x: float, y: float) -> tuple[float, float]:
487 | """Convert screen coordinates to screenshot coordinates.
488 |
489 | Args:
490 | x: X coordinate in screen space
491 | y: Y coordinate in screen space
492 |
493 | Returns:
494 | tuple[float, float]: (x, y) coordinates in screenshot space
495 | """
496 | screen_size = await self.get_screen_size()
497 | screenshot = await self.screenshot()
498 | screenshot_img = bytes_to_image(screenshot)
499 | screenshot_width, screenshot_height = screenshot_img.size
500 |
501 | # Calculate scaling factors
502 | width_scale = screenshot_width / screen_size["width"]
503 | height_scale = screenshot_height / screen_size["height"]
504 |
505 | # Convert coordinates
506 | screenshot_x = x * width_scale
507 | screenshot_y = y * height_scale
508 |
509 | return screenshot_x, screenshot_y
510 |
511 | # Websocket Methods
512 | async def _keep_alive(self):
513 | """Keep the WebSocket connection alive with automatic reconnection."""
514 | retry_count = 0
515 | max_log_attempts = 1 # Only log the first attempt at INFO level
516 | log_interval = 500 # Then log every 500th attempt (significantly increased from 30)
517 | last_warning_time = 0
518 | min_warning_interval = 30 # Minimum seconds between connection lost warnings
519 | min_retry_delay = 0.5 # Minimum delay between connection attempts (500ms)
520 |
521 | while not self._closed:
522 | try:
523 | if self._ws is None or (
524 | self._ws and self._ws.state == websockets.protocol.State.CLOSED
525 | ):
526 | try:
527 | retry_count += 1
528 |
529 | # Add a minimum delay between connection attempts to avoid flooding
530 | if retry_count > 1:
531 | await asyncio.sleep(min_retry_delay)
532 |
533 | # Only log the first attempt at INFO level, then every Nth attempt
534 | if retry_count == 1:
535 | self.logger.info(f"Attempting WebSocket connection to {self.ws_uri}")
536 | elif retry_count % log_interval == 0:
537 | self.logger.info(
538 | f"Still attempting WebSocket connection (attempt {retry_count})..."
539 | )
540 | else:
541 | # All other attempts are logged at DEBUG level
542 | self.logger.debug(
543 | f"Attempting WebSocket connection to {self.ws_uri} (attempt {retry_count})"
544 | )
545 |
546 | self._ws = await asyncio.wait_for(
547 | websockets.connect(
548 | self.ws_uri,
549 | max_size=1024 * 1024 * 10, # 10MB limit
550 | max_queue=32,
551 | ping_interval=self._ping_interval,
552 | ping_timeout=self._ping_timeout,
553 | close_timeout=5,
554 | compression=None, # Disable compression to reduce overhead
555 | ),
556 | timeout=120,
557 | )
558 | self.logger.info("WebSocket connection established")
559 |
560 | # If api_key and vm_name are provided, perform authentication handshake
561 | if self.api_key and self.vm_name:
562 | self.logger.info("Performing authentication handshake...")
563 | auth_message = {
564 | "command": "authenticate",
565 | "params": {
566 | "api_key": self.api_key,
567 | "container_name": self.vm_name
568 | }
569 | }
570 | await self._ws.send(json.dumps(auth_message))
571 |
572 | # Wait for authentication response
573 | async with self._recv_lock:
574 | auth_response = await asyncio.wait_for(self._ws.recv(), timeout=10)
575 | auth_result = json.loads(auth_response)
576 |
577 | if not auth_result.get("success"):
578 | error_msg = auth_result.get("error", "Authentication failed")
579 | self.logger.error(f"Authentication failed: {error_msg}")
580 | await self._ws.close()
581 | self._ws = None
582 | raise ConnectionError(f"Authentication failed: {error_msg}")
583 |
584 | self.logger.info("Authentication successful")
585 |
586 | self._reconnect_delay = 1 # Reset reconnect delay on successful connection
587 | self._last_ping = time.time()
588 | retry_count = 0 # Reset retry count on successful connection
589 | except (asyncio.TimeoutError, websockets.exceptions.WebSocketException) as e:
590 | next_retry = self._reconnect_delay
591 |
592 | # Only log the first error at WARNING level, then every Nth attempt
593 | if retry_count == 1:
594 | self.logger.warning(
595 | f"Computer API Server not ready yet. Will retry automatically."
596 | )
597 | elif retry_count % log_interval == 0:
598 | self.logger.warning(
599 | f"Still waiting for Computer API Server (attempt {retry_count})..."
600 | )
601 | else:
602 | # All other errors are logged at DEBUG level
603 | self.logger.debug(f"Connection attempt {retry_count} failed: {e}")
604 |
605 | if self._ws:
606 | try:
607 | await self._ws.close()
608 | except:
609 | pass
610 | self._ws = None
611 |
612 | # Use exponential backoff for connection retries
613 | await asyncio.sleep(self._reconnect_delay)
614 | self._reconnect_delay = min(
615 | self._reconnect_delay * 2, self._max_reconnect_delay
616 | )
617 | continue
618 |
619 | # Regular ping to check connection
620 | if self._ws and self._ws.state == websockets.protocol.State.OPEN:
621 | try:
622 | if time.time() - self._last_ping >= self._ping_interval:
623 | pong_waiter = await self._ws.ping()
624 | await asyncio.wait_for(pong_waiter, timeout=self._ping_timeout)
625 | self._last_ping = time.time()
626 | except Exception as e:
627 | self.logger.debug(f"Ping failed: {e}")
628 | if self._ws:
629 | try:
630 | await self._ws.close()
631 | except:
632 | pass
633 | self._ws = None
634 | continue
635 |
636 | await asyncio.sleep(1)
637 |
638 | except Exception as e:
639 | current_time = time.time()
640 | # Only log connection lost warnings at most once every min_warning_interval seconds
641 | if current_time - last_warning_time >= min_warning_interval:
642 | self.logger.warning(
643 | f"Computer API Server connection lost. Will retry automatically."
644 | )
645 | last_warning_time = current_time
646 | else:
647 | # Log at debug level instead
648 | self.logger.debug(f"Connection lost: {e}")
649 |
650 | if self._ws:
651 | try:
652 | await self._ws.close()
653 | except:
654 | pass
655 | self._ws = None
656 |
657 | async def _ensure_connection(self):
658 | """Ensure WebSocket connection is established."""
659 | if self._reconnect_task is None or self._reconnect_task.done():
660 | self._reconnect_task = asyncio.create_task(self._keep_alive())
661 |
662 | retry_count = 0
663 | max_retries = 5
664 |
665 | while retry_count < max_retries:
666 | try:
667 | if self._ws and self._ws.state == websockets.protocol.State.OPEN:
668 | return
669 | retry_count += 1
670 | await asyncio.sleep(1)
671 | except Exception as e:
672 | # Only log at ERROR level for the last retry attempt
673 | if retry_count == max_retries - 1:
674 | self.logger.error(
675 | f"Persistent connection check error after {retry_count} attempts: {e}"
676 | )
677 | else:
678 | self.logger.debug(f"Connection check error (attempt {retry_count}): {e}")
679 | retry_count += 1
680 | await asyncio.sleep(1)
681 | continue
682 |
683 | raise ConnectionError("Failed to establish WebSocket connection after multiple retries")
684 |
685 | async def _send_command_ws(self, command: str, params: Optional[Dict] = None) -> Dict[str, Any]:
686 | """Send command through WebSocket."""
687 | max_retries = 3
688 | retry_count = 0
689 | last_error = None
690 |
691 | # Acquire lock to ensure only one command is processed at a time
692 | self.logger.debug(f"Acquired lock for command: {command}")
693 | while retry_count < max_retries:
694 | try:
695 | await self._ensure_connection()
696 | if not self._ws:
697 | raise ConnectionError("WebSocket connection is not established")
698 |
699 | message = {"command": command, "params": params or {}}
700 | await self._ws.send(json.dumps(message))
701 | async with self._recv_lock:
702 | response = await asyncio.wait_for(self._ws.recv(), timeout=120)
703 | self.logger.debug(f"Completed command: {command}")
704 | return json.loads(response)
705 | except Exception as e:
706 | last_error = e
707 | retry_count += 1
708 | if retry_count < max_retries:
709 | # Only log at debug level for intermediate retries
710 | self.logger.debug(
711 | f"Command '{command}' failed (attempt {retry_count}/{max_retries}): {e}"
712 | )
713 | await asyncio.sleep(1)
714 | continue
715 | else:
716 | # Only log at error level for the final failure
717 | self.logger.error(
718 | f"Failed to send command '{command}' after {max_retries} retries"
719 | )
720 | self.logger.debug(f"Command failure details: {e}")
721 | raise
722 |
723 | raise last_error if last_error else RuntimeError("Failed to send command")
724 |
725 | async def _send_command_rest(self, command: str, params: Optional[Dict] = None) -> Dict[str, Any]:
726 | """Send command through REST API without retries or connection management."""
727 | try:
728 | # Prepare the request payload
729 | payload = {"command": command, "params": params or {}}
730 |
731 | # Prepare headers
732 | headers = {"Content-Type": "application/json"}
733 | if self.api_key:
734 | headers["X-API-Key"] = self.api_key
735 | if self.vm_name:
736 | headers["X-Container-Name"] = self.vm_name
737 |
738 | # Send the request
739 | async with aiohttp.ClientSession() as session:
740 | async with session.post(
741 | self.rest_uri,
742 | json=payload,
743 | headers=headers
744 | ) as response:
745 | # Get the response text
746 | response_text = await response.text()
747 |
748 | # Trim whitespace
749 | response_text = response_text.strip()
750 |
751 | # Check if it starts with "data: "
752 | if response_text.startswith("data: "):
753 | # Extract everything after "data: "
754 | json_str = response_text[6:] # Remove "data: " prefix
755 | try:
756 | return json.loads(json_str)
757 | except json.JSONDecodeError:
758 | return {
759 | "success": False,
760 | "error": "Server returned malformed response",
761 | "message": response_text
762 | }
763 | else:
764 | # Return error response
765 | return {
766 | "success": False,
767 | "error": "Server returned malformed response",
768 | "message": response_text
769 | }
770 |
771 | except Exception as e:
772 | return {
773 | "success": False,
774 | "error": "Request failed",
775 | "message": str(e)
776 | }
777 |
778 | async def _send_command(self, command: str, params: Optional[Dict] = None) -> Dict[str, Any]:
779 | """Send command using REST API with WebSocket fallback."""
780 | # Try REST API first
781 | result = await self._send_command_rest(command, params)
782 |
783 | # If REST failed with "Request failed", try WebSocket as fallback
784 | if not result.get("success", True) and (result.get("error") == "Request failed" or result.get("error") == "Server returned malformed response"):
785 | self.logger.warning(f"REST API failed for command '{command}', trying WebSocket fallback")
786 | try:
787 | return await self._send_command_ws(command, params)
788 | except Exception as e:
789 | self.logger.error(f"WebSocket fallback also failed: {e}")
790 | # Return the original REST error
791 | return result
792 |
793 | return result
794 |
795 | async def wait_for_ready(self, timeout: int = 60, interval: float = 1.0):
796 | """Wait for Computer API Server to be ready by testing version command."""
797 |
798 | # Check if REST API is available
799 | try:
800 | result = await self._send_command_rest("version", {})
801 | assert result.get("success", True)
802 | except Exception as e:
803 | self.logger.debug(f"REST API failed for command 'version', trying WebSocket fallback: {e}")
804 | try:
805 | await self._wait_for_ready_ws(timeout, interval)
806 | return
807 | except Exception as e:
808 | self.logger.debug(f"WebSocket fallback also failed: {e}")
809 | raise e
810 |
811 | start_time = time.time()
812 | last_error = None
813 | attempt_count = 0
814 | progress_interval = 10 # Log progress every 10 seconds
815 | last_progress_time = start_time
816 |
817 | try:
818 | self.logger.info(
819 | f"Waiting for Computer API Server to be ready (timeout: {timeout}s)..."
820 | )
821 |
822 | # Wait for the server to respond to get_screen_size command
823 | while time.time() - start_time < timeout:
824 | try:
825 | attempt_count += 1
826 | current_time = time.time()
827 |
828 | # Log progress periodically without flooding logs
829 | if current_time - last_progress_time >= progress_interval:
830 | elapsed = current_time - start_time
831 | self.logger.info(
832 | f"Still waiting for Computer API Server... (elapsed: {elapsed:.1f}s, attempts: {attempt_count})"
833 | )
834 | last_progress_time = current_time
835 |
836 | # Test the server with a simple get_screen_size command
837 | result = await self._send_command("get_screen_size")
838 | if result.get("success", False):
839 | elapsed = time.time() - start_time
840 | self.logger.info(
841 | f"Computer API Server is ready (after {elapsed:.1f}s, {attempt_count} attempts)"
842 | )
843 | return # Server is ready
844 | else:
845 | last_error = result.get("error", "Unknown error")
846 | self.logger.debug(f"Initial connection command failed: {last_error}")
847 |
848 | except Exception as e:
849 | last_error = e
850 | self.logger.debug(f"Connection attempt {attempt_count} failed: {e}")
851 |
852 | # Wait before trying again
853 | await asyncio.sleep(interval)
854 |
855 | # If we get here, we've timed out
856 | error_msg = f"Could not connect to {self.ip_address} after {timeout} seconds"
857 | if last_error:
858 | error_msg += f": {str(last_error)}"
859 | self.logger.error(error_msg)
860 | raise TimeoutError(error_msg)
861 |
862 | except Exception as e:
863 | if isinstance(e, TimeoutError):
864 | raise
865 | error_msg = f"Error while waiting for server: {str(e)}"
866 | self.logger.error(error_msg)
867 | raise RuntimeError(error_msg)
868 |
869 | async def _wait_for_ready_ws(self, timeout: int = 60, interval: float = 1.0):
870 | """Wait for WebSocket connection to become available."""
871 | start_time = time.time()
872 | last_error = None
873 | attempt_count = 0
874 | progress_interval = 10 # Log progress every 10 seconds
875 | last_progress_time = start_time
876 |
877 | # Disable detailed logging for connection attempts
878 | self._log_connection_attempts = False
879 |
880 | try:
881 | self.logger.info(
882 | f"Waiting for Computer API Server to be ready (timeout: {timeout}s)..."
883 | )
884 |
885 | # Start the keep-alive task if it's not already running
886 | if self._reconnect_task is None or self._reconnect_task.done():
887 | self._reconnect_task = asyncio.create_task(self._keep_alive())
888 |
889 | # Wait for the connection to be established
890 | while time.time() - start_time < timeout:
891 | try:
892 | attempt_count += 1
893 | current_time = time.time()
894 |
895 | # Log progress periodically without flooding logs
896 | if current_time - last_progress_time >= progress_interval:
897 | elapsed = current_time - start_time
898 | self.logger.info(
899 | f"Still waiting for Computer API Server... (elapsed: {elapsed:.1f}s, attempts: {attempt_count})"
900 | )
901 | last_progress_time = current_time
902 |
903 | # Check if we have a connection
904 | if self._ws and self._ws.state == websockets.protocol.State.OPEN:
905 | # Test the connection with a simple command
906 | try:
907 | await self._send_command_ws("get_screen_size")
908 | elapsed = time.time() - start_time
909 | self.logger.info(
910 | f"Computer API Server is ready (after {elapsed:.1f}s, {attempt_count} attempts)"
911 | )
912 | return # Connection is fully working
913 | except Exception as e:
914 | last_error = e
915 | self.logger.debug(f"Connection test failed: {e}")
916 |
917 | # Wait before trying again
918 | await asyncio.sleep(interval)
919 |
920 | except Exception as e:
921 | last_error = e
922 | self.logger.debug(f"Connection attempt {attempt_count} failed: {e}")
923 | await asyncio.sleep(interval)
924 |
925 | # If we get here, we've timed out
926 | error_msg = f"Could not connect to {self.ip_address} after {timeout} seconds"
927 | if last_error:
928 | error_msg += f": {str(last_error)}"
929 | self.logger.error(error_msg)
930 | raise TimeoutError(error_msg)
931 | finally:
932 | # Reset to default logging behavior
933 | self._log_connection_attempts = False
934 |
935 | def close(self):
936 | """Close WebSocket connection.
937 |
938 | Note: In host computer server mode, we leave the connection open
939 | to allow other clients to connect to the same server. The server
940 | will handle cleaning up idle connections.
941 | """
942 | # Only cancel the reconnect task
943 | if self._reconnect_task:
944 | self._reconnect_task.cancel()
945 |
946 | # Don't set closed flag or close websocket by default
947 | # This allows the server to stay connected for other clients
948 | # self._closed = True
949 | # if self._ws:
950 | # asyncio.create_task(self._ws.close())
951 | # self._ws = None
952 |
953 | def force_close(self):
954 | """Force close the WebSocket connection.
955 |
956 | This method should be called when you want to completely
957 | shut down the connection, not just for regular cleanup.
958 | """
959 | self._closed = True
960 | if self._reconnect_task:
961 | self._reconnect_task.cancel()
962 | if self._ws:
963 | asyncio.create_task(self._ws.close())
964 | self._ws = None
965 |
966 |
```
--------------------------------------------------------------------------------
/libs/python/computer/computer/providers/lumier/provider.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Lumier VM provider implementation.
3 |
4 | This provider uses Docker containers running the Lumier image to create
5 | macOS and Linux VMs. It handles VM lifecycle operations through Docker
6 | commands and container management.
7 | """
8 |
9 | import logging
10 | import os
11 | import json
12 | import asyncio
13 | from typing import Dict, List, Optional, Any
14 | import subprocess
15 | import time
16 | import re
17 |
18 | from ..base import BaseVMProvider, VMProviderType
19 | from ..lume_api import (
20 | lume_api_get,
21 | lume_api_run,
22 | lume_api_stop,
23 | lume_api_update
24 | )
25 |
26 | # Setup logging
27 | logger = logging.getLogger(__name__)
28 |
29 | # Check if Docker is available
30 | try:
31 | subprocess.run(["docker", "--version"], capture_output=True, check=True)
32 | HAS_LUMIER = True
33 | except (subprocess.SubprocessError, FileNotFoundError):
34 | HAS_LUMIER = False
35 |
36 |
37 | class LumierProvider(BaseVMProvider):
38 | """
39 | Lumier VM Provider implementation using Docker containers.
40 |
41 | This provider uses Docker to run Lumier containers that can create
42 | macOS and Linux VMs through containerization.
43 | """
44 |
45 | def __init__(
46 | self,
47 | port: Optional[int] = 7777,
48 | host: str = "localhost",
49 | storage: Optional[str] = None, # Can be a path or 'ephemeral'
50 | shared_path: Optional[str] = None,
51 | image: str = "macos-sequoia-cua:latest", # VM image to use
52 | verbose: bool = False,
53 | ephemeral: bool = False,
54 | noVNC_port: Optional[int] = 8006,
55 | ):
56 | """Initialize the Lumier VM Provider.
57 |
58 | Args:
59 | port: Port for the API server (default: 7777)
60 | host: Hostname for the API server (default: localhost)
61 | storage: Path for persistent VM storage
62 | shared_path: Path for shared folder between host and VM
63 | image: VM image to use (e.g. "macos-sequoia-cua:latest")
64 | verbose: Enable verbose logging
65 | ephemeral: Use ephemeral (temporary) storage
66 | noVNC_port: Specific port for noVNC interface (default: 8006)
67 | """
68 | self.host = host
69 | # Always ensure api_port has a valid value (7777 is the default)
70 | self.api_port = 7777 if port is None else port
71 | self.vnc_port = noVNC_port # User-specified noVNC port, will be set in run_vm if provided
72 | self.ephemeral = ephemeral
73 |
74 | # Handle ephemeral storage (temporary directory)
75 | if ephemeral:
76 | self.storage = "ephemeral"
77 | else:
78 | self.storage = storage
79 |
80 | self.shared_path = shared_path
81 | self.image = image # Store the VM image name to use
82 | # The container_name will be set in run_vm using the VM name
83 | self.verbose = verbose
84 | self._container_id = None
85 | self._api_url = None # Will be set after container starts
86 |
87 | @property
88 | def provider_type(self) -> VMProviderType:
89 | """Return the provider type."""
90 | return VMProviderType.LUMIER
91 |
92 | def _parse_memory(self, memory_str: str) -> int:
93 | """Parse memory string to MB integer.
94 |
95 | Examples:
96 | "8GB" -> 8192
97 | "1024MB" -> 1024
98 | "512" -> 512
99 | """
100 | if isinstance(memory_str, int):
101 | return memory_str
102 |
103 | if isinstance(memory_str, str):
104 | # Extract number and unit
105 | match = re.match(r"(\d+)([A-Za-z]*)", memory_str)
106 | if match:
107 | value, unit = match.groups()
108 | value = int(value)
109 | unit = unit.upper()
110 |
111 | if unit == "GB" or unit == "G":
112 | return value * 1024
113 | elif unit == "MB" or unit == "M" or unit == "":
114 | return value
115 |
116 | # Default fallback
117 | logger.warning(f"Could not parse memory string '{memory_str}', using 8GB default")
118 | return 8192 # Default to 8GB
119 |
120 | # Helper methods for interacting with the Lumier API through curl
121 | # These methods handle the various VM operations via API calls
122 |
123 | def _get_curl_error_message(self, return_code: int) -> str:
124 | """Get a descriptive error message for curl return codes.
125 |
126 | Args:
127 | return_code: The curl return code
128 |
129 | Returns:
130 | A descriptive error message
131 | """
132 | # Map common curl error codes to helpful messages
133 | if return_code == 7:
134 | return "Failed to connect - API server is starting up"
135 | elif return_code == 22:
136 | return "HTTP error returned from API server"
137 | elif return_code == 28:
138 | return "Operation timeout - API server is slow to respond"
139 | elif return_code == 52:
140 | return "Empty reply from server - API is starting but not ready"
141 | elif return_code == 56:
142 | return "Network problem during data transfer"
143 | else:
144 | return f"Unknown curl error code: {return_code}"
145 |
146 |
147 | async def get_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]:
148 | """Get VM information by name.
149 |
150 | Args:
151 | name: Name of the VM to get information for
152 | storage: Optional storage path override. If provided, this will be used
153 | instead of the provider's default storage path.
154 |
155 | Returns:
156 | Dictionary with VM information including status, IP address, etc.
157 | """
158 | if not HAS_LUMIER:
159 | logger.error("Docker is not available. Cannot get VM status.")
160 | return {
161 | "name": name,
162 | "status": "unavailable",
163 | "error": "Docker is not available"
164 | }
165 |
166 | # Store the current name for API requests
167 | self.container_name = name
168 |
169 | try:
170 | # Check if the container exists and is running
171 | check_cmd = ["docker", "ps", "-a", "--filter", f"name={name}", "--format", "{{.Status}}"]
172 | check_result = subprocess.run(check_cmd, capture_output=True, text=True)
173 | container_status = check_result.stdout.strip()
174 |
175 | if not container_status:
176 | logger.info(f"Container {name} does not exist. Will create when run_vm is called.")
177 | return {
178 | "name": name,
179 | "status": "not_found",
180 | "message": "Container doesn't exist yet"
181 | }
182 |
183 | # Container exists, check if it's running
184 | is_running = container_status.startswith("Up")
185 |
186 | if not is_running:
187 | logger.info(f"Container {name} exists but is not running. Status: {container_status}")
188 | return {
189 | "name": name,
190 | "status": "stopped",
191 | "container_status": container_status,
192 | }
193 |
194 | # Container is running, get the IP address and API status from Lumier API
195 | logger.info(f"Container {name} is running. Getting VM status from API.")
196 |
197 | # Use the shared lume_api_get function directly
198 | vm_info = lume_api_get(
199 | vm_name=name,
200 | host=self.host,
201 | port=self.api_port,
202 | storage=storage if storage is not None else self.storage,
203 | debug=self.verbose,
204 | verbose=self.verbose
205 | )
206 |
207 | # Check for API errors
208 | if "error" in vm_info:
209 | # Use debug level instead of warning to reduce log noise during polling
210 | logger.debug(f"API request error: {vm_info['error']}")
211 | return {
212 | "name": name,
213 | "status": "running", # Container is running even if API is not responsive
214 | "api_status": "error",
215 | "error": vm_info["error"],
216 | "container_status": container_status
217 | }
218 |
219 | # Process the VM status information
220 | vm_status = vm_info.get("status", "unknown")
221 | vnc_url = vm_info.get("vncUrl", "")
222 | ip_address = vm_info.get("ipAddress", "")
223 |
224 | # IMPORTANT: Always ensure we have a valid IP address for connectivity
225 | # If the API doesn't return an IP address, default to localhost (127.0.0.1)
226 | # This makes the behavior consistent with LumeProvider
227 | if not ip_address and vm_status == "running":
228 | ip_address = "127.0.0.1"
229 | logger.info(f"No IP address returned from API, defaulting to {ip_address}")
230 | vm_info["ipAddress"] = ip_address
231 |
232 | logger.info(f"VM {name} status: {vm_status}")
233 |
234 | if ip_address and vnc_url:
235 | logger.info(f"VM {name} has IP: {ip_address} and VNC URL: {vnc_url}")
236 | elif not ip_address and not vnc_url and vm_status != "running":
237 | # Not running is expected in this case
238 | logger.info(f"VM {name} is not running yet. Status: {vm_status}")
239 | else:
240 | # Missing IP or VNC but status is running - this is unusual but handled with default IP
241 | logger.warning(f"VM {name} is running but missing expected fields. API response: {vm_info}")
242 |
243 | # Return the full status information
244 | return {
245 | "name": name,
246 | "status": vm_status,
247 | "ip_address": ip_address,
248 | "vnc_url": vnc_url,
249 | "api_status": "ok",
250 | "container_status": container_status,
251 | **vm_info # Include all fields from the API response
252 | }
253 | except subprocess.SubprocessError as e:
254 | logger.error(f"Failed to check container status: {e}")
255 | return {
256 | "name": name,
257 | "status": "error",
258 | "error": f"Failed to check container status: {str(e)}"
259 | }
260 |
261 | async def list_vms(self) -> List[Dict[str, Any]]:
262 | """List all VMs managed by this provider.
263 |
264 | For Lumier provider, there is only one VM per container.
265 | """
266 | try:
267 | status = await self.get_vm("default")
268 | return [status] if status.get("status") != "unknown" else []
269 | except Exception as e:
270 | logger.error(f"Failed to list VMs: {e}")
271 | return []
272 |
273 | async def run_vm(self, image: str, name: str, run_opts: Dict[str, Any], storage: Optional[str] = None) -> Dict[str, Any]:
274 | """Run a VM with the given options.
275 |
276 | Args:
277 | image: Name/tag of the image to use
278 | name: Name of the VM to run (used for the container name and Docker image tag)
279 | run_opts: Options for running the VM, including:
280 | - cpu: Number of CPU cores
281 | - memory: Amount of memory (e.g. "8GB")
282 | - noVNC_port: Specific port for noVNC interface
283 |
284 | Returns:
285 | Dictionary with VM status information
286 | """
287 | # Set the container name using the VM name for consistency
288 | self.container_name = name
289 | try:
290 | # First, check if container already exists and remove it
291 | try:
292 | check_cmd = ["docker", "ps", "-a", "--filter", f"name={self.container_name}", "--format", "{{.ID}}"]
293 | check_result = subprocess.run(check_cmd, capture_output=True, text=True)
294 | existing_container = check_result.stdout.strip()
295 |
296 | if existing_container:
297 | logger.info(f"Removing existing container: {self.container_name}")
298 | remove_cmd = ["docker", "rm", "-f", self.container_name]
299 | subprocess.run(remove_cmd, check=True)
300 | except subprocess.CalledProcessError as e:
301 | logger.warning(f"Error removing existing container: {e}")
302 | # Continue anyway, next steps will fail if there's a real problem
303 |
304 | # Prepare the Docker run command
305 | cmd = ["docker", "run", "-d", "--name", self.container_name]
306 |
307 | cmd.extend(["-p", f"{self.vnc_port}:8006"])
308 | logger.debug(f"Using specified noVNC_port: {self.vnc_port}")
309 |
310 | # Set API URL using the API port
311 | self._api_url = f"http://{self.host}:{self.api_port}"
312 |
313 | # Parse memory setting
314 | memory_mb = self._parse_memory(run_opts.get("memory", "8GB"))
315 |
316 | # Add storage volume mount if storage is specified (for persistent VM storage)
317 | if self.storage and self.storage != "ephemeral":
318 | # Create storage directory if it doesn't exist
319 | storage_dir = os.path.abspath(os.path.expanduser(self.storage or ""))
320 | os.makedirs(storage_dir, exist_ok=True)
321 |
322 | # Add volume mount for storage
323 | cmd.extend([
324 | "-v", f"{storage_dir}:/storage",
325 | "-e", f"HOST_STORAGE_PATH={storage_dir}"
326 | ])
327 | logger.debug(f"Using persistent storage at: {storage_dir}")
328 |
329 | # Add shared folder volume mount if shared_path is specified
330 | if self.shared_path:
331 | # Create shared directory if it doesn't exist
332 | shared_dir = os.path.abspath(os.path.expanduser(self.shared_path or ""))
333 | os.makedirs(shared_dir, exist_ok=True)
334 |
335 | # Add volume mount for shared folder
336 | cmd.extend([
337 | "-v", f"{shared_dir}:/shared",
338 | "-e", f"HOST_SHARED_PATH={shared_dir}"
339 | ])
340 | logger.debug(f"Using shared folder at: {shared_dir}")
341 |
342 | # Add environment variables
343 | # Always use the container_name as the VM_NAME for consistency
344 | # Use the VM image passed from the Computer class
345 | logger.debug(f"Using VM image: {self.image}")
346 |
347 | # If ghcr.io is in the image, use the full image name
348 | if "ghcr.io" in self.image:
349 | vm_image = self.image
350 | else:
351 | vm_image = f"ghcr.io/trycua/{self.image}"
352 |
353 | cmd.extend([
354 | "-e", f"VM_NAME={self.container_name}",
355 | "-e", f"VERSION={vm_image}",
356 | "-e", f"CPU_CORES={run_opts.get('cpu', '4')}",
357 | "-e", f"RAM_SIZE={memory_mb}",
358 | ])
359 |
360 | # Specify the Lumier image with the full image name
361 | lumier_image = "trycua/lumier:latest"
362 |
363 | # First check if the image exists locally
364 | try:
365 | logger.debug(f"Checking if Docker image {lumier_image} exists locally...")
366 | check_image_cmd = ["docker", "image", "inspect", lumier_image]
367 | subprocess.run(check_image_cmd, capture_output=True, check=True)
368 | logger.debug(f"Docker image {lumier_image} found locally.")
369 | except subprocess.CalledProcessError:
370 | # Image doesn't exist locally
371 | logger.warning(f"\nWARNING: Docker image {lumier_image} not found locally.")
372 | logger.warning("The system will attempt to pull it from Docker Hub, which may fail if you have network connectivity issues.")
373 | logger.warning("If the Docker pull fails, you may need to manually pull the image first with:")
374 | logger.warning(f" docker pull {lumier_image}\n")
375 |
376 | # Add the image to the command
377 | cmd.append(lumier_image)
378 |
379 | # Print the Docker command for debugging
380 | logger.debug(f"DOCKER COMMAND: {' '.join(cmd)}")
381 |
382 | # Run the container with improved error handling
383 | try:
384 | result = subprocess.run(cmd, capture_output=True, text=True, check=True)
385 | except subprocess.CalledProcessError as e:
386 | if "no route to host" in str(e.stderr).lower() or "failed to resolve reference" in str(e.stderr).lower():
387 | error_msg = (f"Network error while trying to pull Docker image '{lumier_image}'\n"
388 | f"Error: {e.stderr}\n\n"
389 | f"SOLUTION: Please try one of the following:\n"
390 | f"1. Check your internet connection\n"
391 | f"2. Pull the image manually with: docker pull {lumier_image}\n"
392 | f"3. Check if Docker is running properly\n")
393 | logger.error(error_msg)
394 | raise RuntimeError(error_msg)
395 | raise
396 |
397 | # Container started, now check VM status with polling
398 | logger.debug("Container started, checking VM status...")
399 | logger.debug("NOTE: This may take some time while the VM image is being pulled and initialized")
400 |
401 | # Start a background thread to show container logs in real-time
402 | import threading
403 |
404 | def show_container_logs():
405 | # Give the container a moment to start generating logs
406 | time.sleep(1)
407 | logger.debug(f"\n---- CONTAINER LOGS FOR '{name}' (LIVE) ----")
408 | logger.debug("Showing logs as they are generated. Press Ctrl+C to stop viewing logs...\n")
409 |
410 | try:
411 | # Use docker logs with follow option
412 | log_cmd = ["docker", "logs", "--tail", "30", "--follow", name]
413 | process = subprocess.Popen(log_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
414 | text=True, bufsize=1, universal_newlines=True)
415 |
416 | # Read and print logs line by line
417 | for line in process.stdout:
418 | logger.debug(line, end='')
419 |
420 | # Break if process has exited
421 | if process.poll() is not None:
422 | break
423 | except Exception as e:
424 | logger.error(f"\nError showing container logs: {e}")
425 | if self.verbose:
426 | logger.error(f"Error in log streaming thread: {e}")
427 | finally:
428 | logger.debug("\n---- LOG STREAMING ENDED ----")
429 | # Make sure process is terminated
430 | if 'process' in locals() and process.poll() is None:
431 | process.terminate()
432 |
433 | # Start log streaming in a background thread if verbose mode is enabled
434 | log_thread = threading.Thread(target=show_container_logs)
435 | log_thread.daemon = True # Thread will exit when main program exits
436 | log_thread.start()
437 |
438 | # Skip waiting for container readiness and just poll get_vm directly
439 | # Poll the get_vm method indefinitely until the VM is ready with an IP address
440 | attempt = 0
441 | consecutive_errors = 0
442 | vm_running = False
443 |
444 | while True: # Wait indefinitely
445 | try:
446 | # Use longer delays to give the system time to initialize
447 | if attempt > 0:
448 | # Start with 5s delay, then increase gradually up to 30s for later attempts
449 | # But use shorter delays while we're getting API errors
450 | if consecutive_errors > 0 and consecutive_errors < 5:
451 | wait_time = 3 # Use shorter delays when we're getting API errors
452 | else:
453 | wait_time = min(30, 5 + (attempt * 2))
454 |
455 | logger.debug(f"Waiting {wait_time}s before retry #{attempt+1}...")
456 | await asyncio.sleep(wait_time)
457 |
458 | # Try to get VM status
459 | logger.debug(f"Checking VM status (attempt {attempt+1})...")
460 | vm_status = await self.get_vm(name)
461 |
462 | # Check for API errors
463 | if 'error' in vm_status:
464 | consecutive_errors += 1
465 | error_msg = vm_status.get('error', 'Unknown error')
466 |
467 | # Only print a user-friendly status message, not the raw error
468 | # since _lume_api_get already logged the technical details
469 | if consecutive_errors == 1 or attempt % 5 == 0:
470 | if 'Empty reply from server' in error_msg:
471 | logger.info("API server is starting up - container is running, but API isn't fully initialized yet.")
472 | logger.info("This is expected during the initial VM setup - will continue polling...")
473 | else:
474 | # Don't repeat the exact same error message each time
475 | logger.warning(f"API request error (attempt {attempt+1}): {error_msg}")
476 | # Just log that we're still working on it
477 | if attempt > 3:
478 | logger.debug("Still waiting for the API server to become available...")
479 |
480 | # If we're getting errors but container is running, that's normal during startup
481 | if vm_status.get('status') == 'running':
482 | if not vm_running:
483 | logger.info("Container is running, waiting for the VM within it to become fully ready...")
484 | logger.info("This might take a minute while the VM initializes...")
485 | vm_running = True
486 |
487 | # Increase counter and continue
488 | attempt += 1
489 | continue
490 |
491 | # Reset consecutive error counter when we get a successful response
492 | consecutive_errors = 0
493 |
494 | # If the VM is running, check if it has an IP address (which means it's fully ready)
495 | if vm_status.get('status') == 'running':
496 | vm_running = True
497 |
498 | # Check if we have an IP address, which means the VM is fully ready
499 | if 'ip_address' in vm_status and vm_status['ip_address']:
500 | logger.info(f"VM is now fully running with IP: {vm_status.get('ip_address')}")
501 | if 'vnc_url' in vm_status and vm_status['vnc_url']:
502 | logger.info(f"VNC URL: {vm_status.get('vnc_url')}")
503 | return vm_status
504 | else:
505 | logger.debug("VM is running but still initializing network interfaces...")
506 | logger.debug("Waiting for IP address to be assigned...")
507 | else:
508 | # VM exists but might still be starting up
509 | status = vm_status.get('status', 'unknown')
510 | logger.debug(f"VM found but status is: {status}. Continuing to poll...")
511 |
512 | # Increase counter for next iteration's delay calculation
513 | attempt += 1
514 |
515 | # If we reach a very large number of attempts, give a reassuring message but continue
516 | if attempt % 10 == 0:
517 | logger.debug(f"Still waiting after {attempt} attempts. This might take several minutes for first-time setup.")
518 | if not vm_running and attempt >= 20:
519 | logger.warning("\nNOTE: First-time VM initialization can be slow as images are downloaded.")
520 | logger.warning("If this continues for more than 10 minutes, you may want to check:")
521 | logger.warning(" 1. Docker logs with: docker logs " + name)
522 | logger.warning(" 2. If your network can access container registries")
523 | logger.warning("Press Ctrl+C to abort if needed.\n")
524 |
525 | # After 150 attempts (likely over 30-40 minutes), return current status
526 | if attempt >= 150:
527 | logger.debug(f"Reached 150 polling attempts. VM status is: {vm_status.get('status', 'unknown')}")
528 | logger.debug("Returning current VM status, but please check Docker logs if there are issues.")
529 | return vm_status
530 |
531 | except Exception as e:
532 | # Always continue retrying, but with increasing delays
533 | logger.warning(f"Error checking VM status (attempt {attempt+1}): {e}. Will retry.")
534 | consecutive_errors += 1
535 |
536 | # If we've had too many consecutive errors, might be a deeper problem
537 | if consecutive_errors >= 10:
538 | logger.warning(f"\nWARNING: Encountered {consecutive_errors} consecutive errors while checking VM status.")
539 | logger.warning("You may need to check the Docker container logs or restart the process.")
540 | logger.warning(f"Error details: {str(e)}\n")
541 |
542 | # Increase attempt counter for next iteration
543 | attempt += 1
544 |
545 | # After many consecutive errors, add a delay to avoid hammering the system
546 | if attempt > 5:
547 | error_delay = min(30, 10 + attempt)
548 | logger.warning(f"Multiple connection errors, waiting {error_delay}s before next attempt...")
549 | await asyncio.sleep(error_delay)
550 |
551 | except subprocess.CalledProcessError as e:
552 | error_msg = f"Failed to start Lumier container: {e.stderr if hasattr(e, 'stderr') else str(e)}"
553 | logger.error(error_msg)
554 | raise RuntimeError(error_msg)
555 |
556 | async def _wait_for_container_ready(self, container_name: str, timeout: int = 90) -> bool:
557 | """Wait for the Lumier container to be fully ready with a valid API response.
558 |
559 | Args:
560 | container_name: Name of the Docker container to check
561 | timeout: Maximum time to wait in seconds (default: 90 seconds)
562 |
563 | Returns:
564 | True if the container is running, even if API is not fully ready.
565 | This allows operations to continue with appropriate fallbacks.
566 | """
567 | start_time = time.time()
568 | api_ready = False
569 | container_running = False
570 |
571 | logger.debug(f"Waiting for container {container_name} to be ready (timeout: {timeout}s)...")
572 |
573 | while time.time() - start_time < timeout:
574 | # Check if container is running
575 | try:
576 | check_cmd = ["docker", "ps", "--filter", f"name={container_name}", "--format", "{{.Status}}"]
577 | result = subprocess.run(check_cmd, capture_output=True, text=True, check=True)
578 | container_status = result.stdout.strip()
579 |
580 | if container_status and container_status.startswith("Up"):
581 | container_running = True
582 | logger.info(f"Container {container_name} is running with status: {container_status}")
583 | else:
584 | logger.warning(f"Container {container_name} not yet running, status: {container_status}")
585 | # container is not running yet, wait and try again
586 | await asyncio.sleep(2) # Longer sleep to give Docker time
587 | continue
588 | except subprocess.CalledProcessError as e:
589 | logger.warning(f"Error checking container status: {e}")
590 | await asyncio.sleep(2)
591 | continue
592 |
593 | # Container is running, check if API is responsive
594 | try:
595 | # First check the health endpoint
596 | api_url = f"http://{self.host}:{self.api_port}/health"
597 | logger.info(f"Checking API health at: {api_url}")
598 |
599 | # Use longer timeout for API health check since it may still be initializing
600 | curl_cmd = ["curl", "-s", "--connect-timeout", "5", "--max-time", "10", api_url]
601 | result = subprocess.run(curl_cmd, capture_output=True, text=True)
602 |
603 | if result.returncode == 0 and "ok" in result.stdout.lower():
604 | api_ready = True
605 | logger.info(f"API is ready at {api_url}")
606 | break
607 | else:
608 | # API health check failed, now let's check if the VM status endpoint is responsive
609 | # This covers cases where the health endpoint isn't implemented but the VM API is working
610 | vm_api_url = f"http://{self.host}:{self.api_port}/lume/vms/{container_name}"
611 | if self.storage:
612 | import urllib.parse
613 | encoded_storage = urllib.parse.quote_plus(self.storage)
614 | vm_api_url += f"?storage={encoded_storage}"
615 |
616 | curl_vm_cmd = ["curl", "-s", "--connect-timeout", "5", "--max-time", "10", vm_api_url]
617 | vm_result = subprocess.run(curl_vm_cmd, capture_output=True, text=True)
618 |
619 | if vm_result.returncode == 0 and vm_result.stdout.strip():
620 | # VM API responded with something - consider the API ready
621 | api_ready = True
622 | logger.info(f"VM API is ready at {vm_api_url}")
623 | break
624 | else:
625 | curl_code = result.returncode
626 | if curl_code == 0:
627 | curl_code = vm_result.returncode
628 |
629 | # Map common curl error codes to helpful messages
630 | if curl_code == 7:
631 | curl_error = "Failed to connect - API server is starting up"
632 | elif curl_code == 22:
633 | curl_error = "HTTP error returned from API server"
634 | elif curl_code == 28:
635 | curl_error = "Operation timeout - API server is slow to respond"
636 | elif curl_code == 52:
637 | curl_error = "Empty reply from server - API is starting but not ready"
638 | elif curl_code == 56:
639 | curl_error = "Network problem during data transfer"
640 | else:
641 | curl_error = f"Unknown curl error code: {curl_code}"
642 |
643 | logger.info(f"API not ready yet: {curl_error}")
644 | except subprocess.SubprocessError as e:
645 | logger.warning(f"Error checking API status: {e}")
646 |
647 | # If the container is running but API is not ready, that's OK - we'll just wait
648 | # a bit longer before checking again, as the container may still be initializing
649 | elapsed_seconds = time.time() - start_time
650 | if int(elapsed_seconds) % 5 == 0: # Only print status every 5 seconds to reduce verbosity
651 | logger.debug(f"Waiting for API to initialize... ({elapsed_seconds:.1f}s / {timeout}s)")
652 |
653 | await asyncio.sleep(3) # Longer sleep between API checks
654 |
655 | # Handle timeout - if the container is running but API is not ready, that's not
656 | # necessarily an error - the API might just need more time to start up
657 | if not container_running:
658 | logger.warning(f"Timed out waiting for container {container_name} to start")
659 | return False
660 |
661 | if not api_ready:
662 | logger.warning(f"Container {container_name} is running, but API is not fully ready yet.")
663 | logger.warning(f"NOTE: You may see some 'API request failed' messages while the API initializes.")
664 |
665 | # Return True if container is running, even if API isn't ready yet
666 | # This allows VM operations to proceed, with appropriate retries for API calls
667 | return container_running
668 |
669 | async def stop_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]:
670 | """Stop a running VM by stopping the Lumier container."""
671 | try:
672 | # Use Docker commands to stop the container directly
673 | if hasattr(self, '_container_id') and self._container_id:
674 | logger.info(f"Stopping Lumier container: {self.container_name}")
675 | cmd = ["docker", "stop", self.container_name]
676 | result = subprocess.run(cmd, capture_output=True, text=True, check=True)
677 | logger.info(f"Container stopped: {result.stdout.strip()}")
678 |
679 | # Return minimal status info
680 | return {
681 | "name": name,
682 | "status": "stopped",
683 | "container_id": self._container_id,
684 | }
685 | else:
686 | # Try to find the container by name
687 | check_cmd = ["docker", "ps", "-a", "--filter", f"name={self.container_name}", "--format", "{{.ID}}"]
688 | check_result = subprocess.run(check_cmd, capture_output=True, text=True)
689 | container_id = check_result.stdout.strip()
690 |
691 | if container_id:
692 | logger.info(f"Found container ID: {container_id}")
693 | cmd = ["docker", "stop", self.container_name]
694 | result = subprocess.run(cmd, capture_output=True, text=True, check=True)
695 | logger.info(f"Container stopped: {result.stdout.strip()}")
696 |
697 | return {
698 | "name": name,
699 | "status": "stopped",
700 | "container_id": container_id,
701 | }
702 | else:
703 | logger.warning(f"No container found with name {self.container_name}")
704 | return {
705 | "name": name,
706 | "status": "unknown",
707 | }
708 | except subprocess.CalledProcessError as e:
709 | error_msg = f"Failed to stop container: {e.stderr if hasattr(e, 'stderr') else str(e)}"
710 | logger.error(error_msg)
711 | raise RuntimeError(f"Failed to stop Lumier container: {error_msg}")
712 |
713 | # update_vm is not implemented as it's not needed for Lumier
714 | # The BaseVMProvider requires it, so we provide a minimal implementation
715 | async def update_vm(self, name: str, update_opts: Dict[str, Any], storage: Optional[str] = None) -> Dict[str, Any]:
716 | """Not implemented for Lumier provider."""
717 | logger.warning("update_vm is not implemented for Lumier provider")
718 | return {"name": name, "status": "unchanged"}
719 |
720 | async def get_logs(self, name: str, num_lines: int = 100, follow: bool = False, timeout: Optional[int] = None) -> str:
721 | """Get the logs from the Lumier container.
722 |
723 | Args:
724 | name: Name of the VM/container to get logs for
725 | num_lines: Number of recent log lines to return (default: 100)
726 | follow: If True, follow the logs (stream new logs as they are generated)
727 | timeout: Optional timeout in seconds for follow mode (None means no timeout)
728 |
729 | Returns:
730 | Container logs as a string
731 |
732 | Note:
733 | If follow=True, this function will continuously stream logs until timeout
734 | or until interrupted. The output will be printed to console in real-time.
735 | """
736 | if not HAS_LUMIER:
737 | error_msg = "Docker is not available. Cannot get container logs."
738 | logger.error(error_msg)
739 | return error_msg
740 |
741 | # Make sure we have a container name
742 | container_name = name
743 |
744 | # Check if the container exists and is running
745 | try:
746 | # Check if the container exists
747 | inspect_cmd = ["docker", "container", "inspect", container_name]
748 | result = subprocess.run(inspect_cmd, capture_output=True, text=True)
749 |
750 | if result.returncode != 0:
751 | error_msg = f"Container '{container_name}' does not exist or is not accessible"
752 | logger.error(error_msg)
753 | return error_msg
754 | except Exception as e:
755 | error_msg = f"Error checking container status: {str(e)}"
756 | logger.error(error_msg)
757 | return error_msg
758 |
759 | # Base docker logs command
760 | log_cmd = ["docker", "logs"]
761 |
762 | # Add tail parameter to limit the number of lines
763 | log_cmd.extend(["--tail", str(num_lines)])
764 |
765 | # Handle follow mode with or without timeout
766 | if follow:
767 | log_cmd.append("--follow")
768 |
769 | if timeout is not None:
770 | # For follow mode with timeout, we'll run the command and handle the timeout
771 | log_cmd.append(container_name)
772 | logger.info(f"Following logs for container '{container_name}' with timeout {timeout}s")
773 | logger.info(f"\n---- CONTAINER LOGS FOR '{container_name}' (LIVE) ----")
774 | logger.info(f"Press Ctrl+C to stop following logs\n")
775 |
776 | try:
777 | # Run with timeout
778 | process = subprocess.Popen(log_cmd, text=True)
779 |
780 | # Wait for the specified timeout
781 | if timeout:
782 | try:
783 | process.wait(timeout=timeout)
784 | except subprocess.TimeoutExpired:
785 | process.terminate() # Stop after timeout
786 | logger.info(f"\n---- LOG FOLLOWING STOPPED (timeout {timeout}s reached) ----")
787 | else:
788 | # Without timeout, wait for user interruption
789 | process.wait()
790 |
791 | return "Logs were displayed to console in follow mode"
792 | except KeyboardInterrupt:
793 | process.terminate()
794 | logger.info("\n---- LOG FOLLOWING STOPPED (user interrupted) ----")
795 | return "Logs were displayed to console in follow mode (interrupted)"
796 | else:
797 | # For follow mode without timeout, we'll print a helpful message
798 | log_cmd.append(container_name)
799 | logger.info(f"Following logs for container '{container_name}' indefinitely")
800 | logger.info(f"\n---- CONTAINER LOGS FOR '{container_name}' (LIVE) ----")
801 | logger.info(f"Press Ctrl+C to stop following logs\n")
802 |
803 | try:
804 | # Run the command and let it run until interrupted
805 | process = subprocess.Popen(log_cmd, text=True)
806 | process.wait() # Wait indefinitely (until user interrupts)
807 | return "Logs were displayed to console in follow mode"
808 | except KeyboardInterrupt:
809 | process.terminate()
810 | logger.info("\n---- LOG FOLLOWING STOPPED (user interrupted) ----")
811 | return "Logs were displayed to console in follow mode (interrupted)"
812 | else:
813 | # For non-follow mode, capture and return the logs as a string
814 | log_cmd.append(container_name)
815 | logger.info(f"Getting {num_lines} log lines for container '{container_name}'")
816 |
817 | try:
818 | result = subprocess.run(log_cmd, capture_output=True, text=True, check=True)
819 | logs = result.stdout
820 |
821 | # Only print header and logs if there's content
822 | if logs.strip():
823 | logger.info(f"\n---- CONTAINER LOGS FOR '{container_name}' (LAST {num_lines} LINES) ----\n")
824 | logger.info(logs)
825 | logger.info(f"\n---- END OF LOGS ----")
826 | else:
827 | logger.info(f"\nNo logs available for container '{container_name}'")
828 |
829 | return logs
830 | except subprocess.CalledProcessError as e:
831 | error_msg = f"Error getting logs: {e.stderr}"
832 | logger.error(error_msg)
833 | return error_msg
834 | except Exception as e:
835 | error_msg = f"Unexpected error getting logs: {str(e)}"
836 | logger.error(error_msg)
837 | return error_msg
838 |
839 | async def restart_vm(self, name: str, storage: Optional[str] = None) -> Dict[str, Any]:
840 | raise NotImplementedError("LumierProvider does not support restarting VMs.")
841 |
842 | async def get_ip(self, name: str, storage: Optional[str] = None, retry_delay: int = 2) -> str:
843 | """Get the IP address of a VM, waiting indefinitely until it's available.
844 |
845 | Args:
846 | name: Name of the VM to get the IP for
847 | storage: Optional storage path override
848 | retry_delay: Delay between retries in seconds (default: 2)
849 |
850 | Returns:
851 | IP address of the VM when it becomes available
852 | """
853 | # Use container_name = name for consistency
854 | self.container_name = name
855 |
856 | # Track total attempts for logging purposes
857 | total_attempts = 0
858 |
859 | # Loop indefinitely until we get a valid IP
860 | while True:
861 | total_attempts += 1
862 |
863 | # Log retry message but not on first attempt
864 | if total_attempts > 1:
865 | logger.info(f"Waiting for VM {name} IP address (attempt {total_attempts})...")
866 |
867 | try:
868 | # Get VM information
869 | vm_info = await self.get_vm(name, storage=storage)
870 |
871 | # Check if we got a valid IP
872 | ip = vm_info.get("ip_address", None)
873 | if ip and ip != "unknown" and not ip.startswith("0.0.0.0"):
874 | logger.info(f"Got valid VM IP address: {ip}")
875 | return ip
876 |
877 | # Check the VM status
878 | status = vm_info.get("status", "unknown")
879 |
880 | # Special handling for Lumier: it may report "stopped" even when the VM is starting
881 | # If the VM information contains an IP but status is stopped, it might be a race condition
882 | if status == "stopped" and "ip_address" in vm_info:
883 | ip = vm_info.get("ip_address")
884 | if ip and ip != "unknown" and not ip.startswith("0.0.0.0"):
885 | logger.info(f"Found valid IP {ip} despite VM status being {status}")
886 | return ip
887 | logger.info(f"VM status is {status}, but still waiting for IP to be assigned")
888 | # If VM is not running yet, log and wait
889 | elif status != "running":
890 | logger.info(f"VM is not running yet (status: {status}). Waiting...")
891 | # If VM is running but no IP yet, wait and retry
892 | else:
893 | logger.info("VM is running but no valid IP address yet. Waiting...")
894 |
895 | except Exception as e:
896 | logger.warning(f"Error getting VM {name} IP: {e}, continuing to wait...")
897 |
898 | # Wait before next retry
899 | await asyncio.sleep(retry_delay)
900 |
901 | # Add progress log every 10 attempts
902 | if total_attempts % 10 == 0:
903 | logger.info(f"Still waiting for VM {name} IP after {total_attempts} attempts...")
904 |
905 | async def __aenter__(self):
906 | """Async context manager entry.
907 |
908 | This method is called when entering an async context manager block.
909 | Returns self to be used in the context.
910 | """
911 | logger.debug("Entering LumierProvider context")
912 |
913 | # Initialize the API URL with the default value if not already set
914 | # This ensures get_vm can work before run_vm is called
915 | if not hasattr(self, '_api_url') or not self._api_url:
916 | self._api_url = f"http://{self.host}:{self.api_port}"
917 | logger.info(f"Initialized default Lumier API URL: {self._api_url}")
918 |
919 | return self
920 |
921 | async def __aexit__(self, exc_type, exc_val, exc_tb):
922 | """Async context manager exit.
923 |
924 | This method is called when exiting an async context manager block.
925 | It handles proper cleanup of resources, including stopping any running containers.
926 | """
927 | logger.debug(f"Exiting LumierProvider context, handling exceptions: {exc_type}")
928 | try:
929 | # If we have a container ID, we should stop it to clean up resources
930 | if hasattr(self, '_container_id') and self._container_id:
931 | logger.info(f"Stopping Lumier container on context exit: {self.container_name}")
932 | try:
933 | cmd = ["docker", "stop", self.container_name]
934 | subprocess.run(cmd, capture_output=True, text=True, check=True)
935 | logger.info(f"Container stopped during context exit: {self.container_name}")
936 | except subprocess.CalledProcessError as e:
937 | logger.warning(f"Failed to stop container during cleanup: {e.stderr}")
938 | # Don't raise an exception here, we want to continue with cleanup
939 | except Exception as e:
940 | logger.error(f"Error during LumierProvider cleanup: {e}")
941 | # We don't want to suppress the original exception if there was one
942 | if exc_type is None:
943 | raise
944 | # Return False to indicate that any exception should propagate
945 | return False
946 |
```
--------------------------------------------------------------------------------
/libs/python/computer/computer/computer.py:
--------------------------------------------------------------------------------
```python
1 | import traceback
2 | from typing import Optional, List, Literal, Dict, Any, Union, TYPE_CHECKING, cast
3 | import asyncio
4 | from .models import Computer as ComputerConfig, Display
5 | from .interface.factory import InterfaceFactory
6 | import time
7 | from PIL import Image
8 | import io
9 | import re
10 | from .logger import Logger, LogLevel
11 | import json
12 | import logging
13 | from core.telemetry import is_telemetry_enabled, record_event
14 | import os
15 | from . import helpers
16 |
17 | import platform
18 |
19 | SYSTEM_INFO = {
20 | "os": platform.system().lower(),
21 | "os_version": platform.release(),
22 | "python_version": platform.python_version(),
23 | }
24 |
25 | # Import provider related modules
26 | from .providers.base import VMProviderType
27 | from .providers.factory import VMProviderFactory
28 |
29 | OSType = Literal["macos", "linux", "windows"]
30 |
31 | class Computer:
32 | """Computer is the main class for interacting with the computer."""
33 |
34 | def create_desktop_from_apps(self, apps):
35 | """
36 | Create a virtual desktop from a list of app names, returning a DioramaComputer
37 | that proxies Diorama.Interface but uses diorama_cmds via the computer interface.
38 |
39 | Args:
40 | apps (list[str]): List of application names to include in the desktop.
41 | Returns:
42 | DioramaComputer: A proxy object with the Diorama interface, but using diorama_cmds.
43 | """
44 | assert "app-use" in self.experiments, "App Usage is an experimental feature. Enable it by passing experiments=['app-use'] to Computer()"
45 | from .diorama_computer import DioramaComputer
46 | return DioramaComputer(self, apps)
47 |
48 | def __init__(
49 | self,
50 | display: Union[Display, Dict[str, int], str] = "1024x768",
51 | memory: str = "8GB",
52 | cpu: str = "4",
53 | os_type: OSType = "macos",
54 | name: str = "",
55 | image: Optional[str] = None,
56 | shared_directories: Optional[List[str]] = None,
57 | use_host_computer_server: bool = False,
58 | verbosity: Union[int, LogLevel] = logging.INFO,
59 | telemetry_enabled: bool = True,
60 | provider_type: Union[str, VMProviderType] = VMProviderType.LUME,
61 | port: Optional[int] = 7777,
62 | noVNC_port: Optional[int] = 8006,
63 | host: str = os.environ.get("PYLUME_HOST", "localhost"),
64 | storage: Optional[str] = None,
65 | ephemeral: bool = False,
66 | api_key: Optional[str] = None,
67 | experiments: Optional[List[str]] = None
68 | ):
69 | """Initialize a new Computer instance.
70 |
71 | Args:
72 | display: The display configuration. Can be:
73 | - A Display object
74 | - A dict with 'width' and 'height'
75 | - A string in format "WIDTHxHEIGHT" (e.g. "1920x1080")
76 | Defaults to "1024x768"
77 | memory: The VM memory allocation. Defaults to "8GB"
78 | cpu: The VM CPU allocation. Defaults to "4"
79 | os_type: The operating system type ('macos' or 'linux')
80 | name: The VM name
81 | image: The VM image name
82 | shared_directories: Optional list of directory paths to share with the VM
83 | use_host_computer_server: If True, target localhost instead of starting a VM
84 | verbosity: Logging level (standard Python logging levels: logging.DEBUG, logging.INFO, etc.)
85 | LogLevel enum values are still accepted for backward compatibility
86 | telemetry_enabled: Whether to enable telemetry tracking. Defaults to True.
87 | provider_type: The VM provider type to use (lume, qemu, cloud)
88 | port: Optional port to use for the VM provider server
89 | noVNC_port: Optional port for the noVNC web interface (Lumier provider)
90 | host: Host to use for VM provider connections (e.g. "localhost", "host.docker.internal")
91 | storage: Optional path for persistent VM storage (Lumier provider)
92 | ephemeral: Whether to use ephemeral storage
93 | api_key: Optional API key for cloud providers
94 | experiments: Optional list of experimental features to enable (e.g. ["app-use"])
95 | """
96 |
97 | self.logger = Logger("computer", verbosity)
98 | self.logger.info("Initializing Computer...")
99 |
100 | if not image:
101 | if os_type == "macos":
102 | image = "macos-sequoia-cua:latest"
103 | elif os_type == "linux":
104 | image = "trycua/cua-ubuntu:latest"
105 | image = str(image)
106 |
107 | # Store original parameters
108 | self.image = image
109 | self.port = port
110 | self.noVNC_port = noVNC_port
111 | self.host = host
112 | self.os_type = os_type
113 | self.provider_type = provider_type
114 | self.ephemeral = ephemeral
115 |
116 | self.api_key = api_key
117 | self.experiments = experiments or []
118 |
119 | if "app-use" in self.experiments:
120 | assert self.os_type == "macos", "App use experiment is only supported on macOS"
121 |
122 | # The default is currently to use non-ephemeral storage
123 | if storage and ephemeral and storage != "ephemeral":
124 | raise ValueError("Storage path and ephemeral flag cannot be used together")
125 |
126 | # Windows Sandbox always uses ephemeral storage
127 | if self.provider_type == VMProviderType.WINSANDBOX:
128 | if not ephemeral and storage != None and storage != "ephemeral":
129 | self.logger.warning("Windows Sandbox storage is always ephemeral. Setting ephemeral=True.")
130 | self.ephemeral = True
131 | self.storage = "ephemeral"
132 | else:
133 | self.storage = "ephemeral" if ephemeral else storage
134 |
135 | # For Lumier provider, store the first shared directory path to use
136 | # for VM file sharing
137 | self.shared_path = None
138 | if shared_directories and len(shared_directories) > 0:
139 | self.shared_path = shared_directories[0]
140 | self.logger.info(f"Using first shared directory for VM file sharing: {self.shared_path}")
141 |
142 | # Store telemetry preference
143 | self._telemetry_enabled = telemetry_enabled
144 |
145 | # Set initialization flag
146 | self._initialized = False
147 | self._running = False
148 |
149 | # Configure root logger
150 | self.verbosity = verbosity
151 | self.logger = Logger("computer", verbosity)
152 |
153 | # Configure component loggers with proper hierarchy
154 | self.vm_logger = Logger("computer.vm", verbosity)
155 | self.interface_logger = Logger("computer.interface", verbosity)
156 |
157 | if not use_host_computer_server:
158 | if ":" not in image:
159 | image = f"{image}:latest"
160 |
161 | if not name:
162 | # Normalize the name to be used for the VM
163 | name = image.replace(":", "_")
164 | # Remove any forward slashes
165 | name = name.replace("/", "_")
166 |
167 | # Convert display parameter to Display object
168 | if isinstance(display, str):
169 | # Parse string format "WIDTHxHEIGHT"
170 | match = re.match(r"(\d+)x(\d+)", display)
171 | if not match:
172 | raise ValueError(
173 | "Display string must be in format 'WIDTHxHEIGHT' (e.g. '1024x768')"
174 | )
175 | width, height = map(int, match.groups())
176 | display_config = Display(width=width, height=height)
177 | elif isinstance(display, dict):
178 | display_config = Display(**display)
179 | else:
180 | display_config = display
181 |
182 | self.config = ComputerConfig(
183 | image=image.split(":")[0],
184 | tag=image.split(":")[1],
185 | name=name,
186 | display=display_config,
187 | memory=memory,
188 | cpu=cpu,
189 | )
190 | # Initialize VM provider but don't start it yet - we'll do that in run()
191 | self.config.vm_provider = None # Will be initialized in run()
192 |
193 | # Store shared directories config
194 | self.shared_directories = shared_directories or []
195 |
196 | # Placeholder for VM provider context manager
197 | self._provider_context = None
198 |
199 | # Initialize with proper typing - None at first, will be set in run()
200 | self._interface = None
201 | self.use_host_computer_server = use_host_computer_server
202 |
203 | # Record initialization in telemetry (if enabled)
204 | if telemetry_enabled and is_telemetry_enabled():
205 | record_event("computer_initialized", SYSTEM_INFO)
206 | else:
207 | self.logger.debug("Telemetry disabled - skipping initialization tracking")
208 |
209 | async def __aenter__(self):
210 | """Start the computer."""
211 | await self.run()
212 | return self
213 |
214 | async def __aexit__(self, exc_type, exc_val, exc_tb):
215 | """Stop the computer."""
216 | await self.disconnect()
217 |
218 | def __enter__(self):
219 | """Start the computer."""
220 | # Run the event loop to call the async enter method
221 | loop = asyncio.get_event_loop()
222 | loop.run_until_complete(self.__aenter__())
223 | return self
224 |
225 | def __exit__(self, exc_type, exc_val, exc_tb):
226 | """Stop the computer."""
227 | loop = asyncio.get_event_loop()
228 | loop.run_until_complete(self.__aexit__(exc_type, exc_val, exc_tb))
229 |
230 | async def run(self) -> Optional[str]:
231 | """Initialize the VM and computer interface."""
232 | if TYPE_CHECKING:
233 | from .interface.base import BaseComputerInterface
234 |
235 | # If already initialized, just log and return
236 | if hasattr(self, "_initialized") and self._initialized:
237 | self.logger.info("Computer already initialized, skipping initialization")
238 | return
239 |
240 | self.logger.info("Starting computer...")
241 | start_time = time.time()
242 |
243 | try:
244 | # If using host computer server
245 | if self.use_host_computer_server:
246 | self.logger.info("Using host computer server")
247 | # Set ip_address for host computer server mode
248 | ip_address = "localhost"
249 | # Create the interface with explicit type annotation
250 | from .interface.base import BaseComputerInterface
251 |
252 | self._interface = cast(
253 | BaseComputerInterface,
254 | InterfaceFactory.create_interface_for_os(
255 | os=self.os_type, ip_address=ip_address # type: ignore[arg-type]
256 | ),
257 | )
258 |
259 | self.logger.info("Waiting for host computer server to be ready...")
260 | await self._interface.wait_for_ready()
261 | self.logger.info("Host computer server ready")
262 | else:
263 | # Start or connect to VM
264 | self.logger.info(f"Starting VM: {self.image}")
265 | if not self._provider_context:
266 | try:
267 | provider_type_name = self.provider_type.name if isinstance(self.provider_type, VMProviderType) else self.provider_type
268 | self.logger.verbose(f"Initializing {provider_type_name} provider context...")
269 |
270 | # Explicitly set provider parameters
271 | storage = "ephemeral" if self.ephemeral else self.storage
272 | verbose = self.verbosity >= LogLevel.DEBUG
273 | ephemeral = self.ephemeral
274 | port = self.port if self.port is not None else 7777
275 | host = self.host if self.host else "localhost"
276 | image = self.image
277 | shared_path = self.shared_path
278 | noVNC_port = self.noVNC_port
279 |
280 | # Create VM provider instance with explicit parameters
281 | try:
282 | if self.provider_type == VMProviderType.LUMIER:
283 | self.logger.info(f"Using VM image for Lumier provider: {image}")
284 | if shared_path:
285 | self.logger.info(f"Using shared path for Lumier provider: {shared_path}")
286 | if noVNC_port:
287 | self.logger.info(f"Using noVNC port for Lumier provider: {noVNC_port}")
288 | self.config.vm_provider = VMProviderFactory.create_provider(
289 | self.provider_type,
290 | port=port,
291 | host=host,
292 | storage=storage,
293 | shared_path=shared_path,
294 | image=image,
295 | verbose=verbose,
296 | ephemeral=ephemeral,
297 | noVNC_port=noVNC_port,
298 | )
299 | elif self.provider_type == VMProviderType.LUME:
300 | self.config.vm_provider = VMProviderFactory.create_provider(
301 | self.provider_type,
302 | port=port,
303 | host=host,
304 | storage=storage,
305 | verbose=verbose,
306 | ephemeral=ephemeral,
307 | )
308 | elif self.provider_type == VMProviderType.CLOUD:
309 | self.config.vm_provider = VMProviderFactory.create_provider(
310 | self.provider_type,
311 | api_key=self.api_key,
312 | verbose=verbose,
313 | )
314 | elif self.provider_type == VMProviderType.WINSANDBOX:
315 | self.config.vm_provider = VMProviderFactory.create_provider(
316 | self.provider_type,
317 | port=port,
318 | host=host,
319 | storage=storage,
320 | verbose=verbose,
321 | ephemeral=ephemeral,
322 | noVNC_port=noVNC_port,
323 | )
324 | elif self.provider_type == VMProviderType.DOCKER:
325 | self.config.vm_provider = VMProviderFactory.create_provider(
326 | self.provider_type,
327 | port=port,
328 | host=host,
329 | storage=storage,
330 | shared_path=shared_path,
331 | image=image or "trycua/cua-ubuntu:latest",
332 | verbose=verbose,
333 | ephemeral=ephemeral,
334 | noVNC_port=noVNC_port,
335 | )
336 | else:
337 | raise ValueError(f"Unsupported provider type: {self.provider_type}")
338 | self._provider_context = await self.config.vm_provider.__aenter__()
339 | self.logger.verbose("VM provider context initialized successfully")
340 | except ImportError as ie:
341 | self.logger.error(f"Failed to import provider dependencies: {ie}")
342 | if str(ie).find("lume") >= 0 and str(ie).find("lumier") < 0:
343 | self.logger.error("Please install with: pip install cua-computer[lume]")
344 | elif str(ie).find("lumier") >= 0 or str(ie).find("docker") >= 0:
345 | self.logger.error("Please install with: pip install cua-computer[lumier] and make sure Docker is installed")
346 | elif str(ie).find("cloud") >= 0:
347 | self.logger.error("Please install with: pip install cua-computer[cloud]")
348 | raise
349 | except Exception as e:
350 | self.logger.error(f"Failed to initialize provider context: {e}")
351 | raise RuntimeError(f"Failed to initialize VM provider: {e}")
352 |
353 | # Check if VM exists or create it
354 | is_running = False
355 | try:
356 | if self.config.vm_provider is None:
357 | raise RuntimeError(f"VM provider not initialized for {self.config.name}")
358 |
359 | vm = await self.config.vm_provider.get_vm(self.config.name)
360 | self.logger.verbose(f"Found existing VM: {self.config.name}")
361 | is_running = vm.get("status") == "running"
362 | except Exception as e:
363 | self.logger.error(f"VM not found: {self.config.name}")
364 | self.logger.error(f"Error: {e}")
365 | raise RuntimeError(
366 | f"VM {self.config.name} could not be found or created."
367 | )
368 |
369 | # Start the VM if it's not running
370 | if not is_running:
371 | self.logger.info(f"VM {self.config.name} is not running, starting it...")
372 |
373 | # Convert paths to dictionary format for shared directories
374 | shared_dirs = []
375 | for path in self.shared_directories:
376 | self.logger.verbose(f"Adding shared directory: {path}")
377 | path = os.path.abspath(os.path.expanduser(path))
378 | if os.path.exists(path):
379 | # Add path in format expected by Lume API
380 | shared_dirs.append({
381 | "hostPath": path,
382 | "readOnly": False
383 | })
384 | else:
385 | self.logger.warning(f"Shared directory does not exist: {path}")
386 |
387 | # Prepare run options to pass to the provider
388 | run_opts = {}
389 |
390 | # Add display information if available
391 | if self.config.display is not None:
392 | display_info = {
393 | "width": self.config.display.width,
394 | "height": self.config.display.height,
395 | }
396 |
397 | # Check if scale_factor exists before adding it
398 | if hasattr(self.config.display, "scale_factor"):
399 | display_info["scale_factor"] = self.config.display.scale_factor
400 |
401 | run_opts["display"] = display_info
402 |
403 | # Add shared directories if available
404 | if self.shared_directories:
405 | run_opts["shared_directories"] = shared_dirs.copy()
406 |
407 | # Run the VM with the provider
408 | try:
409 | if self.config.vm_provider is None:
410 | raise RuntimeError(f"VM provider not initialized for {self.config.name}")
411 |
412 | # Use the complete run_opts we prepared earlier
413 | # Handle ephemeral storage for run_vm method too
414 | storage_param = "ephemeral" if self.ephemeral else self.storage
415 |
416 | # Log the image being used
417 | self.logger.info(f"Running VM using image: {self.image}")
418 |
419 | # Call provider.run_vm with explicit image parameter
420 | response = await self.config.vm_provider.run_vm(
421 | image=self.image,
422 | name=self.config.name,
423 | run_opts=run_opts,
424 | storage=storage_param
425 | )
426 | self.logger.info(f"VM run response: {response if response else 'None'}")
427 | except Exception as run_error:
428 | self.logger.error(f"Failed to run VM: {run_error}")
429 | raise RuntimeError(f"Failed to start VM: {run_error}")
430 |
431 | # Wait for VM to be ready with a valid IP address
432 | self.logger.info("Waiting for VM to be ready with a valid IP address...")
433 | try:
434 | if self.provider_type == VMProviderType.LUMIER:
435 | max_retries = 60 # Increased for Lumier VM startup which takes longer
436 | retry_delay = 3 # 3 seconds between retries for Lumier
437 | else:
438 | max_retries = 30 # Default for other providers
439 | retry_delay = 2 # 2 seconds between retries
440 |
441 | self.logger.info(f"Waiting up to {max_retries * retry_delay} seconds for VM to be ready...")
442 | ip = await self.get_ip(max_retries=max_retries, retry_delay=retry_delay)
443 |
444 | # If we get here, we have a valid IP
445 | self.logger.info(f"VM is ready with IP: {ip}")
446 | ip_address = ip
447 | except TimeoutError as timeout_error:
448 | self.logger.error(str(timeout_error))
449 | raise RuntimeError(f"VM startup timed out: {timeout_error}")
450 | except Exception as wait_error:
451 | self.logger.error(f"Error waiting for VM: {wait_error}")
452 | raise RuntimeError(f"VM failed to become ready: {wait_error}")
453 | except Exception as e:
454 | self.logger.error(f"Failed to initialize computer: {e}")
455 | self.logger.error(traceback.format_exc())
456 | raise RuntimeError(f"Failed to initialize computer: {e}")
457 |
458 | try:
459 | # Verify we have a valid IP before initializing the interface
460 | if not ip_address or ip_address == "unknown" or ip_address == "0.0.0.0":
461 | raise RuntimeError(f"Cannot initialize interface - invalid IP address: {ip_address}")
462 |
463 | # Initialize the interface using the factory with the specified OS
464 | self.logger.info(f"Initializing interface for {self.os_type} at {ip_address}")
465 | from .interface.base import BaseComputerInterface
466 |
467 | # Pass authentication credentials if using cloud provider
468 | if self.provider_type == VMProviderType.CLOUD and self.api_key and self.config.name:
469 | self._interface = cast(
470 | BaseComputerInterface,
471 | InterfaceFactory.create_interface_for_os(
472 | os=self.os_type,
473 | ip_address=ip_address,
474 | api_key=self.api_key,
475 | vm_name=self.config.name
476 | ),
477 | )
478 | else:
479 | self._interface = cast(
480 | BaseComputerInterface,
481 | InterfaceFactory.create_interface_for_os(
482 | os=self.os_type,
483 | ip_address=ip_address
484 | ),
485 | )
486 |
487 | # Wait for the WebSocket interface to be ready
488 | self.logger.info("Connecting to WebSocket interface...")
489 |
490 | try:
491 | # Use a single timeout for the entire connection process
492 | # The VM should already be ready at this point, so we're just establishing the connection
493 | await self._interface.wait_for_ready(timeout=30)
494 | self.logger.info("WebSocket interface connected successfully")
495 | except TimeoutError as e:
496 | self.logger.error(f"Failed to connect to WebSocket interface at {ip_address}")
497 | raise TimeoutError(
498 | f"Could not connect to WebSocket interface at {ip_address}:8000/ws: {str(e)}"
499 | )
500 | # self.logger.warning(
501 | # f"Could not connect to WebSocket interface at {ip_address}:8000/ws: {str(e)}, expect missing functionality"
502 | # )
503 |
504 | # Create an event to keep the VM running in background if needed
505 | if not self.use_host_computer_server:
506 | self._stop_event = asyncio.Event()
507 | self._keep_alive_task = asyncio.create_task(self._stop_event.wait())
508 |
509 | self.logger.info("Computer is ready")
510 |
511 | # Set the initialization flag and clear the initializing flag
512 | self._initialized = True
513 |
514 | # Set this instance as the default computer for remote decorators
515 | helpers.set_default_computer(self)
516 |
517 | self.logger.info("Computer successfully initialized")
518 | except Exception as e:
519 | raise
520 | finally:
521 | # Log initialization time for performance monitoring
522 | duration_ms = (time.time() - start_time) * 1000
523 | self.logger.debug(f"Computer initialization took {duration_ms:.2f}ms")
524 | return
525 |
526 | async def disconnect(self) -> None:
527 | """Disconnect from the computer's WebSocket interface."""
528 | if self._interface:
529 | self._interface.close()
530 |
531 | async def stop(self) -> None:
532 | """Disconnect from the computer's WebSocket interface and stop the computer."""
533 | start_time = time.time()
534 |
535 | try:
536 | self.logger.info("Stopping Computer...")
537 |
538 | # In VM mode, first explicitly stop the VM, then exit the provider context
539 | if not self.use_host_computer_server and self._provider_context and self.config.vm_provider is not None:
540 | try:
541 | self.logger.info(f"Stopping VM {self.config.name}...")
542 | await self.config.vm_provider.stop_vm(
543 | name=self.config.name,
544 | storage=self.storage # Pass storage explicitly for clarity
545 | )
546 | except Exception as e:
547 | self.logger.error(f"Error stopping VM: {e}")
548 |
549 | self.logger.verbose("Closing VM provider context...")
550 | await self.config.vm_provider.__aexit__(None, None, None)
551 | self._provider_context = None
552 |
553 | await self.disconnect()
554 | self.logger.info("Computer stopped")
555 | except Exception as e:
556 | self.logger.debug(f"Error during cleanup: {e}") # Log as debug since this might be expected
557 | finally:
558 | # Log stop time for performance monitoring
559 | duration_ms = (time.time() - start_time) * 1000
560 | self.logger.debug(f"Computer stop process took {duration_ms:.2f}ms")
561 | return
562 |
563 | async def start(self) -> None:
564 | """Start the computer."""
565 | await self.run()
566 |
567 | async def restart(self) -> None:
568 | """Restart the computer.
569 |
570 | If using a VM provider that supports restart, this will issue a restart
571 | without tearing down the provider context, then reconnect the interface.
572 | Falls back to stop()+run() when a provider restart is not available.
573 | """
574 | # Host computer server: just disconnect and run again
575 | if self.use_host_computer_server:
576 | try:
577 | await self.disconnect()
578 | finally:
579 | await self.run()
580 | return
581 |
582 | # If no VM provider context yet, fall back to full run
583 | if not getattr(self, "_provider_context", None) or self.config.vm_provider is None:
584 | self.logger.info("No provider context active; performing full restart via run()")
585 | await self.run()
586 | return
587 |
588 | # Gracefully close current interface connection if present
589 | if self._interface:
590 | try:
591 | self._interface.close()
592 | except Exception as e:
593 | self.logger.debug(f"Error closing interface prior to restart: {e}")
594 |
595 | # Attempt provider-level restart if implemented
596 | try:
597 | storage_param = "ephemeral" if self.ephemeral else self.storage
598 | if hasattr(self.config.vm_provider, "restart_vm"):
599 | self.logger.info(f"Restarting VM {self.config.name} via provider...")
600 | await self.config.vm_provider.restart_vm(name=self.config.name, storage=storage_param)
601 | else:
602 | # Fallback: stop then start without leaving provider context
603 | self.logger.info(f"Provider has no restart_vm; performing stop+start for {self.config.name}...")
604 | await self.config.vm_provider.stop_vm(name=self.config.name, storage=storage_param)
605 | await self.config.vm_provider.run_vm(image=self.image, name=self.config.name, run_opts={}, storage=storage_param)
606 | except Exception as e:
607 | self.logger.error(f"Failed to restart VM via provider: {e}")
608 | # As a last resort, do a full stop (with provider context exit) and run
609 | try:
610 | await self.stop()
611 | finally:
612 | await self.run()
613 | return
614 |
615 | # Wait for VM to be ready and reconnect interface
616 | try:
617 | self.logger.info("Waiting for VM to be ready after restart...")
618 | if self.provider_type == VMProviderType.LUMIER:
619 | max_retries = 60
620 | retry_delay = 3
621 | else:
622 | max_retries = 30
623 | retry_delay = 2
624 | ip_address = await self.get_ip(max_retries=max_retries, retry_delay=retry_delay)
625 |
626 | self.logger.info(f"Re-initializing interface for {self.os_type} at {ip_address}")
627 | from .interface.base import BaseComputerInterface
628 |
629 | if self.provider_type == VMProviderType.CLOUD and self.api_key and self.config.name:
630 | self._interface = cast(
631 | BaseComputerInterface,
632 | InterfaceFactory.create_interface_for_os(
633 | os=self.os_type,
634 | ip_address=ip_address,
635 | api_key=self.api_key,
636 | vm_name=self.config.name,
637 | ),
638 | )
639 | else:
640 | self._interface = cast(
641 | BaseComputerInterface,
642 | InterfaceFactory.create_interface_for_os(
643 | os=self.os_type,
644 | ip_address=ip_address,
645 | ),
646 | )
647 |
648 | self.logger.info("Connecting to WebSocket interface after restart...")
649 | await self._interface.wait_for_ready(timeout=30)
650 | self.logger.info("Computer reconnected and ready after restart")
651 | except Exception as e:
652 | self.logger.error(f"Failed to reconnect after restart: {e}")
653 | # Try a full reset if reconnection failed
654 | try:
655 | await self.stop()
656 | finally:
657 | await self.run()
658 |
659 | # @property
660 | async def get_ip(self, max_retries: int = 15, retry_delay: int = 3) -> str:
661 | """Get the IP address of the VM or localhost if using host computer server.
662 |
663 | This method delegates to the provider's get_ip method, which waits indefinitely
664 | until the VM has a valid IP address.
665 |
666 | Args:
667 | max_retries: Unused parameter, kept for backward compatibility
668 | retry_delay: Delay between retries in seconds (default: 2)
669 |
670 | Returns:
671 | IP address of the VM or localhost if using host computer server
672 | """
673 | # For host computer server, always return localhost immediately
674 | if self.use_host_computer_server:
675 | return "127.0.0.1"
676 |
677 | # Get IP from the provider - each provider implements its own waiting logic
678 | if self.config.vm_provider is None:
679 | raise RuntimeError("VM provider is not initialized")
680 |
681 | # Log that we're waiting for the IP
682 | self.logger.info(f"Waiting for VM {self.config.name} to get an IP address...")
683 |
684 | # Call the provider's get_ip method which will wait indefinitely
685 | storage_param = "ephemeral" if self.ephemeral else self.storage
686 |
687 | # Log the image being used
688 | self.logger.info(f"Running VM using image: {self.image}")
689 |
690 | # Call provider.get_ip with explicit image parameter
691 | ip = await self.config.vm_provider.get_ip(
692 | name=self.config.name,
693 | storage=storage_param,
694 | retry_delay=retry_delay
695 | )
696 |
697 | # Log success
698 | self.logger.info(f"VM {self.config.name} has IP address: {ip}")
699 | return ip
700 |
701 |
702 | async def wait_vm_ready(self) -> Optional[Dict[str, Any]]:
703 | """Wait for VM to be ready with an IP address.
704 |
705 | Returns:
706 | VM status information or None if using host computer server.
707 | """
708 | if self.use_host_computer_server:
709 | return None
710 |
711 | timeout = 600 # 10 minutes timeout (increased from 4 minutes)
712 | interval = 2.0 # 2 seconds between checks (increased to reduce API load)
713 | start_time = time.time()
714 | last_status = None
715 | attempts = 0
716 |
717 | self.logger.info(f"Waiting for VM {self.config.name} to be ready (timeout: {timeout}s)...")
718 |
719 | while time.time() - start_time < timeout:
720 | attempts += 1
721 | elapsed = time.time() - start_time
722 |
723 | try:
724 | # Keep polling for VM info
725 | if self.config.vm_provider is None:
726 | self.logger.error("VM provider is not initialized")
727 | vm = None
728 | else:
729 | vm = await self.config.vm_provider.get_vm(self.config.name)
730 |
731 | # Log full VM properties for debugging (every 30 attempts)
732 | if attempts % 30 == 0:
733 | self.logger.info(
734 | f"VM properties at attempt {attempts}: {vars(vm) if vm else 'None'}"
735 | )
736 |
737 | # Get current status for logging
738 | current_status = getattr(vm, "status", None) if vm else None
739 | if current_status != last_status:
740 | self.logger.info(
741 | f"VM status changed to: {current_status} (after {elapsed:.1f}s)"
742 | )
743 | last_status = current_status
744 |
745 | # Check for IP address - ensure it's not None or empty
746 | ip = getattr(vm, "ip_address", None) if vm else None
747 | if ip and ip.strip(): # Check for non-empty string
748 | self.logger.info(
749 | f"VM {self.config.name} got IP address: {ip} (after {elapsed:.1f}s)"
750 | )
751 | return vm
752 |
753 | if attempts % 10 == 0: # Log every 10 attempts to avoid flooding
754 | self.logger.info(
755 | f"Still waiting for VM IP address... (elapsed: {elapsed:.1f}s)"
756 | )
757 | else:
758 | self.logger.debug(
759 | f"Waiting for VM IP address... Current IP: {ip}, Status: {current_status}"
760 | )
761 |
762 | except Exception as e:
763 | self.logger.warning(f"Error checking VM status (attempt {attempts}): {str(e)}")
764 | # If we've been trying for a while and still getting errors, log more details
765 | if elapsed > 60: # After 1 minute of errors, log more details
766 | self.logger.error(f"Persistent error getting VM status: {str(e)}")
767 | self.logger.info("Trying to get VM list for debugging...")
768 | try:
769 | if self.config.vm_provider is not None:
770 | vms = await self.config.vm_provider.list_vms()
771 | self.logger.info(
772 | f"Available VMs: {[getattr(vm, 'name', None) for vm in vms if hasattr(vm, 'name')]}"
773 | )
774 | except Exception as list_error:
775 | self.logger.error(f"Failed to list VMs: {str(list_error)}")
776 |
777 | await asyncio.sleep(interval)
778 |
779 | # If we get here, we've timed out
780 | elapsed = time.time() - start_time
781 | self.logger.error(f"VM {self.config.name} not ready after {elapsed:.1f} seconds")
782 |
783 | # Try to get final VM status for debugging
784 | try:
785 | if self.config.vm_provider is not None:
786 | vm = await self.config.vm_provider.get_vm(self.config.name)
787 | # VM data is returned as a dictionary from the Lumier provider
788 | status = vm.get('status', 'unknown') if vm else "unknown"
789 | ip = vm.get('ip_address') if vm else None
790 | else:
791 | status = "unknown"
792 | ip = None
793 | self.logger.error(f"Final VM status: {status}, IP: {ip}")
794 | except Exception as e:
795 | self.logger.error(f"Failed to get final VM status: {str(e)}")
796 |
797 | raise TimeoutError(
798 | f"VM {self.config.name} not ready after {elapsed:.1f} seconds - IP address not assigned"
799 | )
800 |
801 | async def update(self, cpu: Optional[int] = None, memory: Optional[str] = None):
802 | """Update VM settings."""
803 | self.logger.info(
804 | f"Updating VM settings: CPU={cpu or self.config.cpu}, Memory={memory or self.config.memory}"
805 | )
806 | update_opts = {
807 | "cpu": cpu or int(self.config.cpu),
808 | "memory": memory or self.config.memory
809 | }
810 | if self.config.vm_provider is not None:
811 | await self.config.vm_provider.update_vm(
812 | name=self.config.name,
813 | update_opts=update_opts,
814 | storage=self.storage # Pass storage explicitly for clarity
815 | )
816 | else:
817 | raise RuntimeError("VM provider not initialized")
818 |
819 | def get_screenshot_size(self, screenshot: bytes) -> Dict[str, int]:
820 | """Get the dimensions of a screenshot.
821 |
822 | Args:
823 | screenshot: The screenshot bytes
824 |
825 | Returns:
826 | Dict[str, int]: Dictionary containing 'width' and 'height' of the image
827 | """
828 | image = Image.open(io.BytesIO(screenshot))
829 | width, height = image.size
830 | return {"width": width, "height": height}
831 |
832 | @property
833 | def interface(self):
834 | """Get the computer interface for interacting with the VM.
835 |
836 | Returns:
837 | The computer interface
838 | """
839 | if not hasattr(self, "_interface") or self._interface is None:
840 | error_msg = "Computer interface not initialized. Call run() first."
841 | self.logger.error(error_msg)
842 | self.logger.error(
843 | "Make sure to call await computer.run() before using any interface methods."
844 | )
845 | raise RuntimeError(error_msg)
846 |
847 | return self._interface
848 |
849 | @property
850 | def telemetry_enabled(self) -> bool:
851 | """Check if telemetry is enabled for this computer instance.
852 |
853 | Returns:
854 | bool: True if telemetry is enabled, False otherwise
855 | """
856 | return self._telemetry_enabled
857 |
858 | async def to_screen_coordinates(self, x: float, y: float) -> tuple[float, float]:
859 | """Convert normalized coordinates to screen coordinates.
860 |
861 | Args:
862 | x: X coordinate between 0 and 1
863 | y: Y coordinate between 0 and 1
864 |
865 | Returns:
866 | tuple[float, float]: Screen coordinates (x, y)
867 | """
868 | return await self.interface.to_screen_coordinates(x, y)
869 |
870 | async def to_screenshot_coordinates(self, x: float, y: float) -> tuple[float, float]:
871 | """Convert screen coordinates to screenshot coordinates.
872 |
873 | Args:
874 | x: X coordinate in screen space
875 | y: Y coordinate in screen space
876 |
877 | Returns:
878 | tuple[float, float]: (x, y) coordinates in screenshot space
879 | """
880 | return await self.interface.to_screenshot_coordinates(x, y)
881 |
882 |
883 | # Add virtual environment management functions to computer interface
884 | async def venv_install(self, venv_name: str, requirements: list[str]):
885 | """Install packages in a virtual environment.
886 |
887 | Args:
888 | venv_name: Name of the virtual environment
889 | requirements: List of package requirements to install
890 |
891 | Returns:
892 | Tuple of (stdout, stderr) from the installation command
893 | """
894 | requirements = requirements or []
895 | # Windows vs POSIX handling
896 | if self.os_type == "windows":
897 | # Use %USERPROFILE% for home directory and cmd.exe semantics
898 | venv_path = f"%USERPROFILE%\\.venvs\\{venv_name}"
899 | ensure_dir_cmd = "if not exist \"%USERPROFILE%\\.venvs\" mkdir \"%USERPROFILE%\\.venvs\""
900 | create_cmd = f"if not exist \"{venv_path}\" python -m venv \"{venv_path}\""
901 | requirements_str = " ".join(requirements)
902 | # Activate via activate.bat and install
903 | install_cmd = f"call \"{venv_path}\\Scripts\\activate.bat\" && pip install {requirements_str}" if requirements_str else f"echo No requirements to install"
904 | await self.interface.run_command(ensure_dir_cmd)
905 | await self.interface.run_command(create_cmd)
906 | return await self.interface.run_command(install_cmd)
907 | else:
908 | # POSIX (macOS/Linux)
909 | venv_path = f"$HOME/.venvs/{venv_name}"
910 | create_cmd = f"mkdir -p \"$HOME/.venvs\" && python3 -m venv \"{venv_path}\""
911 | # Check if venv exists, if not create it
912 | check_cmd = f"test -d \"{venv_path}\" || ({create_cmd})"
913 | _ = await self.interface.run_command(check_cmd)
914 | # Install packages
915 | requirements_str = " ".join(requirements)
916 | install_cmd = (
917 | f". \"{venv_path}/bin/activate\" && pip install {requirements_str}"
918 | if requirements_str
919 | else "echo No requirements to install"
920 | )
921 | return await self.interface.run_command(install_cmd)
922 |
923 | async def venv_cmd(self, venv_name: str, command: str):
924 | """Execute a shell command in a virtual environment.
925 |
926 | Args:
927 | venv_name: Name of the virtual environment
928 | command: Shell command to execute in the virtual environment
929 |
930 | Returns:
931 | Tuple of (stdout, stderr) from the command execution
932 | """
933 | if self.os_type == "windows":
934 | # Windows (cmd.exe)
935 | venv_path = f"%USERPROFILE%\\.venvs\\{venv_name}"
936 | # Check existence and signal if missing
937 | check_cmd = f"if not exist \"{venv_path}\" (echo VENV_NOT_FOUND) else (echo VENV_FOUND)"
938 | result = await self.interface.run_command(check_cmd)
939 | if "VENV_NOT_FOUND" in getattr(result, "stdout", ""):
940 | # Auto-create the venv with no requirements
941 | await self.venv_install(venv_name, [])
942 | # Activate and run the command
943 | full_command = f"call \"{venv_path}\\Scripts\\activate.bat\" && {command}"
944 | return await self.interface.run_command(full_command)
945 | else:
946 | # POSIX (macOS/Linux)
947 | venv_path = f"$HOME/.venvs/{venv_name}"
948 | # Check if virtual environment exists
949 | check_cmd = f"test -d \"{venv_path}\""
950 | result = await self.interface.run_command(check_cmd)
951 | if result.stderr or "test:" in result.stdout: # venv doesn't exist
952 | # Auto-create the venv with no requirements
953 | await self.venv_install(venv_name, [])
954 | # Activate virtual environment and run command
955 | full_command = f". \"{venv_path}/bin/activate\" && {command}"
956 | return await self.interface.run_command(full_command)
957 |
958 | async def venv_exec(self, venv_name: str, python_func, *args, **kwargs):
959 | """Execute Python function in a virtual environment using source code extraction.
960 |
961 | Args:
962 | venv_name: Name of the virtual environment
963 | python_func: A callable function to execute
964 | *args: Positional arguments to pass to the function
965 | **kwargs: Keyword arguments to pass to the function
966 |
967 | Returns:
968 | The result of the function execution, or raises any exception that occurred
969 | """
970 | import base64
971 | import inspect
972 | import json
973 | import textwrap
974 |
975 | try:
976 | # Get function source code using inspect.getsource
977 | source = inspect.getsource(python_func)
978 | # Remove common leading whitespace (dedent)
979 | func_source = textwrap.dedent(source).strip()
980 |
981 | # Remove decorators
982 | while func_source.lstrip().startswith("@"):
983 | func_source = func_source.split("\n", 1)[1].strip()
984 |
985 | # Get function name for execution
986 | func_name = python_func.__name__
987 |
988 | # Serialize args and kwargs as JSON (safer than dill for cross-version compatibility)
989 | args_json = json.dumps(args, default=str)
990 | kwargs_json = json.dumps(kwargs, default=str)
991 |
992 | except OSError as e:
993 | raise Exception(f"Cannot retrieve source code for function {python_func.__name__}: {e}")
994 | except Exception as e:
995 | raise Exception(f"Failed to reconstruct function source: {e}")
996 |
997 | # Create Python code that will define and execute the function
998 | python_code = f'''
999 | import json
1000 | import traceback
1001 |
1002 | try:
1003 | # Define the function from source
1004 | {textwrap.indent(func_source, " ")}
1005 |
1006 | # Deserialize args and kwargs from JSON
1007 | args_json = """{args_json}"""
1008 | kwargs_json = """{kwargs_json}"""
1009 | args = json.loads(args_json)
1010 | kwargs = json.loads(kwargs_json)
1011 |
1012 | # Execute the function
1013 | result = {func_name}(*args, **kwargs)
1014 |
1015 | # Create success output payload
1016 | output_payload = {{
1017 | "success": True,
1018 | "result": result,
1019 | "error": None
1020 | }}
1021 |
1022 | except Exception as e:
1023 | # Create error output payload
1024 | output_payload = {{
1025 | "success": False,
1026 | "result": None,
1027 | "error": {{
1028 | "type": type(e).__name__,
1029 | "message": str(e),
1030 | "traceback": traceback.format_exc()
1031 | }}
1032 | }}
1033 |
1034 | # Serialize the output payload as JSON
1035 | import json
1036 | output_json = json.dumps(output_payload, default=str)
1037 |
1038 | # Print the JSON output with markers
1039 | print(f"<<<VENV_EXEC_START>>>{{output_json}}<<<VENV_EXEC_END>>>")
1040 | '''
1041 |
1042 | # Encode the Python code in base64 to avoid shell escaping issues
1043 | encoded_code = base64.b64encode(python_code.encode('utf-8')).decode('ascii')
1044 |
1045 | # Execute the Python code in the virtual environment
1046 | python_command = f"python -c \"import base64; exec(base64.b64decode('{encoded_code}').decode('utf-8'))\""
1047 | result = await self.venv_cmd(venv_name, python_command)
1048 |
1049 | # Parse the output to extract the payload
1050 | start_marker = "<<<VENV_EXEC_START>>>"
1051 | end_marker = "<<<VENV_EXEC_END>>>"
1052 |
1053 | # Print original stdout
1054 | print(result.stdout[:result.stdout.find(start_marker)])
1055 |
1056 | if start_marker in result.stdout and end_marker in result.stdout:
1057 | start_idx = result.stdout.find(start_marker) + len(start_marker)
1058 | end_idx = result.stdout.find(end_marker)
1059 |
1060 | if start_idx < end_idx:
1061 | output_json = result.stdout[start_idx:end_idx]
1062 |
1063 | try:
1064 | # Decode and deserialize the output payload from JSON
1065 | output_payload = json.loads(output_json)
1066 | except Exception as e:
1067 | raise Exception(f"Failed to decode output payload: {e}")
1068 |
1069 | if output_payload["success"]:
1070 | return output_payload["result"]
1071 | else:
1072 | # Recreate and raise the original exception
1073 | error_info = output_payload["error"]
1074 | error_class = eval(error_info["type"])
1075 | raise error_class(error_info["message"])
1076 | else:
1077 | raise Exception("Invalid output format: markers found but no content between them")
1078 | else:
1079 | # Fallback: return stdout/stderr if no payload markers found
1080 | raise Exception(f"No output payload found. stdout: {result.stdout}, stderr: {result.stderr}")
1081 |
```