This is page 4 of 28. Use http://codebase.md/trycua/cua?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .cursorignore
├── .dockerignore
├── .editorconfig
├── .gitattributes
├── .github
│ ├── FUNDING.yml
│ ├── scripts
│ │ ├── get_pyproject_version.py
│ │ └── tests
│ │ ├── __init__.py
│ │ ├── README.md
│ │ └── test_get_pyproject_version.py
│ └── workflows
│ ├── bump-version.yml
│ ├── ci-lume.yml
│ ├── docker-publish-cua-linux.yml
│ ├── docker-publish-cua-windows.yml
│ ├── docker-publish-kasm.yml
│ ├── docker-publish-xfce.yml
│ ├── docker-reusable-publish.yml
│ ├── link-check.yml
│ ├── lint.yml
│ ├── npm-publish-cli.yml
│ ├── npm-publish-computer.yml
│ ├── npm-publish-core.yml
│ ├── publish-lume.yml
│ ├── pypi-publish-agent.yml
│ ├── pypi-publish-computer-server.yml
│ ├── pypi-publish-computer.yml
│ ├── pypi-publish-core.yml
│ ├── pypi-publish-mcp-server.yml
│ ├── pypi-publish-som.yml
│ ├── pypi-reusable-publish.yml
│ ├── python-tests.yml
│ ├── test-cua-models.yml
│ └── test-validation-script.yml
├── .gitignore
├── .pre-commit-config.yaml
├── .prettierignore
├── .prettierrc.yaml
├── .vscode
│ ├── docs.code-workspace
│ ├── extensions.json
│ ├── launch.json
│ ├── libs-ts.code-workspace
│ ├── lume.code-workspace
│ ├── lumier.code-workspace
│ ├── py.code-workspace
│ └── settings.json
├── blog
│ ├── app-use.md
│ ├── assets
│ │ ├── composite-agents.png
│ │ ├── docker-ubuntu-support.png
│ │ ├── hack-booth.png
│ │ ├── hack-closing-ceremony.jpg
│ │ ├── hack-cua-ollama-hud.jpeg
│ │ ├── hack-leaderboard.png
│ │ ├── hack-the-north.png
│ │ ├── hack-winners.jpeg
│ │ ├── hack-workshop.jpeg
│ │ ├── hud-agent-evals.png
│ │ └── trajectory-viewer.jpeg
│ ├── bringing-computer-use-to-the-web.md
│ ├── build-your-own-operator-on-macos-1.md
│ ├── build-your-own-operator-on-macos-2.md
│ ├── cloud-windows-ga-macos-preview.md
│ ├── composite-agents.md
│ ├── computer-use-agents-for-growth-hacking.md
│ ├── cua-hackathon.md
│ ├── cua-playground-preview.md
│ ├── cua-vlm-router.md
│ ├── hack-the-north.md
│ ├── hud-agent-evals.md
│ ├── human-in-the-loop.md
│ ├── introducing-cua-cli.md
│ ├── introducing-cua-cloud-containers.md
│ ├── lume-to-containerization.md
│ ├── neurips-2025-cua-papers.md
│ ├── sandboxed-python-execution.md
│ ├── training-computer-use-models-trajectories-1.md
│ ├── trajectory-viewer.md
│ ├── ubuntu-docker-support.md
│ └── windows-sandbox.md
├── CONTRIBUTING.md
├── Development.md
├── Dockerfile
├── docs
│ ├── .env.example
│ ├── .gitignore
│ ├── content
│ │ └── docs
│ │ ├── agent-sdk
│ │ │ ├── agent-loops.mdx
│ │ │ ├── benchmarks
│ │ │ │ ├── index.mdx
│ │ │ │ ├── interactive.mdx
│ │ │ │ ├── introduction.mdx
│ │ │ │ ├── meta.json
│ │ │ │ ├── osworld-verified.mdx
│ │ │ │ ├── screenspot-pro.mdx
│ │ │ │ └── screenspot-v2.mdx
│ │ │ ├── callbacks
│ │ │ │ ├── agent-lifecycle.mdx
│ │ │ │ ├── cost-saving.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── logging.mdx
│ │ │ │ ├── meta.json
│ │ │ │ ├── pii-anonymization.mdx
│ │ │ │ └── trajectories.mdx
│ │ │ ├── chat-history.mdx
│ │ │ ├── custom-tools.mdx
│ │ │ ├── customizing-computeragent.mdx
│ │ │ ├── integrations
│ │ │ │ ├── hud.mdx
│ │ │ │ ├── meta.json
│ │ │ │ └── observability.mdx
│ │ │ ├── mcp-server
│ │ │ │ ├── client-integrations.mdx
│ │ │ │ ├── configuration.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── installation.mdx
│ │ │ │ ├── llm-integrations.mdx
│ │ │ │ ├── meta.json
│ │ │ │ ├── tools.mdx
│ │ │ │ └── usage.mdx
│ │ │ ├── message-format.mdx
│ │ │ ├── meta.json
│ │ │ ├── migration-guide.mdx
│ │ │ ├── prompt-caching.mdx
│ │ │ ├── supported-agents
│ │ │ │ ├── composed-agents.mdx
│ │ │ │ ├── computer-use-agents.mdx
│ │ │ │ ├── grounding-models.mdx
│ │ │ │ ├── human-in-the-loop.mdx
│ │ │ │ └── meta.json
│ │ │ ├── supported-model-providers
│ │ │ │ ├── cua-vlm-router.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ └── local-models.mdx
│ │ │ ├── telemetry.mdx
│ │ │ └── usage-tracking.mdx
│ │ ├── cli-playbook
│ │ │ ├── commands.mdx
│ │ │ ├── index.mdx
│ │ │ └── meta.json
│ │ ├── computer-sdk
│ │ │ ├── cloud-vm-management.mdx
│ │ │ ├── commands.mdx
│ │ │ ├── computer-server
│ │ │ │ ├── Commands.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── meta.json
│ │ │ │ ├── REST-API.mdx
│ │ │ │ └── WebSocket-API.mdx
│ │ │ ├── computer-ui.mdx
│ │ │ ├── computers.mdx
│ │ │ ├── custom-computer-handlers.mdx
│ │ │ ├── meta.json
│ │ │ ├── sandboxed-python.mdx
│ │ │ └── tracing-api.mdx
│ │ ├── example-usecases
│ │ │ ├── form-filling.mdx
│ │ │ ├── gemini-complex-ui-navigation.mdx
│ │ │ ├── meta.json
│ │ │ ├── post-event-contact-export.mdx
│ │ │ └── windows-app-behind-vpn.mdx
│ │ ├── get-started
│ │ │ ├── meta.json
│ │ │ └── quickstart.mdx
│ │ ├── index.mdx
│ │ ├── macos-vm-cli-playbook
│ │ │ ├── lume
│ │ │ │ ├── cli-reference.mdx
│ │ │ │ ├── faq.md
│ │ │ │ ├── http-api.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── installation.mdx
│ │ │ │ ├── meta.json
│ │ │ │ └── prebuilt-images.mdx
│ │ │ ├── lumier
│ │ │ │ ├── building-lumier.mdx
│ │ │ │ ├── docker-compose.mdx
│ │ │ │ ├── docker.mdx
│ │ │ │ ├── index.mdx
│ │ │ │ ├── installation.mdx
│ │ │ │ └── meta.json
│ │ │ └── meta.json
│ │ └── meta.json
│ ├── next.config.mjs
│ ├── package-lock.json
│ ├── package.json
│ ├── pnpm-lock.yaml
│ ├── postcss.config.mjs
│ ├── public
│ │ └── img
│ │ ├── agent_gradio_ui.png
│ │ ├── agent.png
│ │ ├── bg-dark.jpg
│ │ ├── bg-light.jpg
│ │ ├── cli.png
│ │ ├── computer.png
│ │ ├── grounding-with-gemini3.gif
│ │ ├── hero.png
│ │ ├── laminar_trace_example.png
│ │ ├── som_box_threshold.png
│ │ └── som_iou_threshold.png
│ ├── README.md
│ ├── source.config.ts
│ ├── src
│ │ ├── app
│ │ │ ├── (home)
│ │ │ │ ├── [[...slug]]
│ │ │ │ │ └── page.tsx
│ │ │ │ └── layout.tsx
│ │ │ ├── api
│ │ │ │ ├── posthog
│ │ │ │ │ └── [...path]
│ │ │ │ │ └── route.ts
│ │ │ │ └── search
│ │ │ │ └── route.ts
│ │ │ ├── favicon.ico
│ │ │ ├── global.css
│ │ │ ├── layout.config.tsx
│ │ │ ├── layout.tsx
│ │ │ ├── llms.mdx
│ │ │ │ └── [[...slug]]
│ │ │ │ └── route.ts
│ │ │ ├── llms.txt
│ │ │ │ └── route.ts
│ │ │ ├── robots.ts
│ │ │ └── sitemap.ts
│ │ ├── assets
│ │ │ ├── discord-black.svg
│ │ │ ├── discord-white.svg
│ │ │ ├── logo-black.svg
│ │ │ └── logo-white.svg
│ │ ├── components
│ │ │ ├── analytics-tracker.tsx
│ │ │ ├── cookie-consent.tsx
│ │ │ ├── doc-actions-menu.tsx
│ │ │ ├── editable-code-block.tsx
│ │ │ ├── footer.tsx
│ │ │ ├── hero.tsx
│ │ │ ├── iou.tsx
│ │ │ ├── mermaid.tsx
│ │ │ └── page-feedback.tsx
│ │ ├── lib
│ │ │ ├── llms.ts
│ │ │ └── source.ts
│ │ ├── mdx-components.tsx
│ │ └── providers
│ │ └── posthog-provider.tsx
│ └── tsconfig.json
├── examples
│ ├── agent_examples.py
│ ├── agent_ui_examples.py
│ ├── browser_tool_example.py
│ ├── cloud_api_examples.py
│ ├── computer_examples_windows.py
│ ├── computer_examples.py
│ ├── computer_ui_examples.py
│ ├── computer-example-ts
│ │ ├── .env.example
│ │ ├── .gitignore
│ │ ├── package-lock.json
│ │ ├── package.json
│ │ ├── pnpm-lock.yaml
│ │ ├── README.md
│ │ ├── src
│ │ │ ├── helpers.ts
│ │ │ └── index.ts
│ │ └── tsconfig.json
│ ├── docker_examples.py
│ ├── evals
│ │ ├── hud_eval_examples.py
│ │ └── wikipedia_most_linked.txt
│ ├── pylume_examples.py
│ ├── sandboxed_functions_examples.py
│ ├── som_examples.py
│ ├── tracing_examples.py
│ ├── utils.py
│ └── winsandbox_example.py
├── img
│ ├── agent_gradio_ui.png
│ ├── agent.png
│ ├── cli.png
│ ├── computer.png
│ ├── logo_black.png
│ └── logo_white.png
├── libs
│ ├── kasm
│ │ ├── Dockerfile
│ │ ├── LICENSE
│ │ ├── README.md
│ │ └── src
│ │ └── ubuntu
│ │ └── install
│ │ └── firefox
│ │ ├── custom_startup.sh
│ │ ├── firefox.desktop
│ │ └── install_firefox.sh
│ ├── lume
│ │ ├── .cursorignore
│ │ ├── CONTRIBUTING.md
│ │ ├── Development.md
│ │ ├── img
│ │ │ └── cli.png
│ │ ├── Package.resolved
│ │ ├── Package.swift
│ │ ├── README.md
│ │ ├── resources
│ │ │ └── lume.entitlements
│ │ ├── scripts
│ │ │ ├── build
│ │ │ │ ├── build-debug.sh
│ │ │ │ ├── build-release-notarized.sh
│ │ │ │ └── build-release.sh
│ │ │ └── install.sh
│ │ ├── src
│ │ │ ├── Commands
│ │ │ │ ├── Clone.swift
│ │ │ │ ├── Config.swift
│ │ │ │ ├── Create.swift
│ │ │ │ ├── Delete.swift
│ │ │ │ ├── Get.swift
│ │ │ │ ├── Images.swift
│ │ │ │ ├── IPSW.swift
│ │ │ │ ├── List.swift
│ │ │ │ ├── Logs.swift
│ │ │ │ ├── Options
│ │ │ │ │ └── FormatOption.swift
│ │ │ │ ├── Prune.swift
│ │ │ │ ├── Pull.swift
│ │ │ │ ├── Push.swift
│ │ │ │ ├── Run.swift
│ │ │ │ ├── Serve.swift
│ │ │ │ ├── Set.swift
│ │ │ │ └── Stop.swift
│ │ │ ├── ContainerRegistry
│ │ │ │ ├── ImageContainerRegistry.swift
│ │ │ │ ├── ImageList.swift
│ │ │ │ └── ImagesPrinter.swift
│ │ │ ├── Errors
│ │ │ │ └── Errors.swift
│ │ │ ├── FileSystem
│ │ │ │ ├── Home.swift
│ │ │ │ ├── Settings.swift
│ │ │ │ ├── VMConfig.swift
│ │ │ │ ├── VMDirectory.swift
│ │ │ │ └── VMLocation.swift
│ │ │ ├── LumeController.swift
│ │ │ ├── Main.swift
│ │ │ ├── Server
│ │ │ │ ├── Handlers.swift
│ │ │ │ ├── HTTP.swift
│ │ │ │ ├── Requests.swift
│ │ │ │ ├── Responses.swift
│ │ │ │ └── Server.swift
│ │ │ ├── Utils
│ │ │ │ ├── CommandRegistry.swift
│ │ │ │ ├── CommandUtils.swift
│ │ │ │ ├── Logger.swift
│ │ │ │ ├── NetworkUtils.swift
│ │ │ │ ├── Path.swift
│ │ │ │ ├── ProcessRunner.swift
│ │ │ │ ├── ProgressLogger.swift
│ │ │ │ ├── String.swift
│ │ │ │ └── Utils.swift
│ │ │ ├── Virtualization
│ │ │ │ ├── DarwinImageLoader.swift
│ │ │ │ ├── DHCPLeaseParser.swift
│ │ │ │ ├── ImageLoaderFactory.swift
│ │ │ │ └── VMVirtualizationService.swift
│ │ │ ├── VM
│ │ │ │ ├── DarwinVM.swift
│ │ │ │ ├── LinuxVM.swift
│ │ │ │ ├── VM.swift
│ │ │ │ ├── VMDetails.swift
│ │ │ │ ├── VMDetailsPrinter.swift
│ │ │ │ ├── VMDisplayResolution.swift
│ │ │ │ └── VMFactory.swift
│ │ │ └── VNC
│ │ │ ├── PassphraseGenerator.swift
│ │ │ └── VNCService.swift
│ │ └── tests
│ │ ├── Mocks
│ │ │ ├── MockVM.swift
│ │ │ ├── MockVMVirtualizationService.swift
│ │ │ └── MockVNCService.swift
│ │ ├── VM
│ │ │ └── VMDetailsPrinterTests.swift
│ │ ├── VMTests.swift
│ │ ├── VMVirtualizationServiceTests.swift
│ │ └── VNCServiceTests.swift
│ ├── lumier
│ │ ├── .dockerignore
│ │ ├── Dockerfile
│ │ ├── README.md
│ │ └── src
│ │ ├── bin
│ │ │ └── entry.sh
│ │ ├── config
│ │ │ └── constants.sh
│ │ ├── hooks
│ │ │ └── on-logon.sh
│ │ └── lib
│ │ ├── utils.sh
│ │ └── vm.sh
│ ├── python
│ │ ├── agent
│ │ │ ├── .bumpversion.cfg
│ │ │ ├── agent
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __main__.py
│ │ │ │ ├── adapters
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── cua_adapter.py
│ │ │ │ │ ├── huggingfacelocal_adapter.py
│ │ │ │ │ ├── human_adapter.py
│ │ │ │ │ ├── mlxvlm_adapter.py
│ │ │ │ │ └── models
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── generic.py
│ │ │ │ │ ├── internvl.py
│ │ │ │ │ ├── opencua.py
│ │ │ │ │ └── qwen2_5_vl.py
│ │ │ │ ├── agent.py
│ │ │ │ ├── callbacks
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── budget_manager.py
│ │ │ │ │ ├── image_retention.py
│ │ │ │ │ ├── logging.py
│ │ │ │ │ ├── operator_validator.py
│ │ │ │ │ ├── pii_anonymization.py
│ │ │ │ │ ├── prompt_instructions.py
│ │ │ │ │ ├── telemetry.py
│ │ │ │ │ └── trajectory_saver.py
│ │ │ │ ├── cli.py
│ │ │ │ ├── computers
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── cua.py
│ │ │ │ │ └── custom.py
│ │ │ │ ├── decorators.py
│ │ │ │ ├── human_tool
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── __main__.py
│ │ │ │ │ ├── server.py
│ │ │ │ │ └── ui.py
│ │ │ │ ├── integrations
│ │ │ │ │ └── hud
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── agent.py
│ │ │ │ │ └── proxy.py
│ │ │ │ ├── loops
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── anthropic.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── composed_grounded.py
│ │ │ │ │ ├── gelato.py
│ │ │ │ │ ├── gemini.py
│ │ │ │ │ ├── generic_vlm.py
│ │ │ │ │ ├── glm45v.py
│ │ │ │ │ ├── gta1.py
│ │ │ │ │ ├── holo.py
│ │ │ │ │ ├── internvl.py
│ │ │ │ │ ├── model_types.csv
│ │ │ │ │ ├── moondream3.py
│ │ │ │ │ ├── omniparser.py
│ │ │ │ │ ├── openai.py
│ │ │ │ │ ├── opencua.py
│ │ │ │ │ ├── uiins.py
│ │ │ │ │ ├── uitars.py
│ │ │ │ │ └── uitars2.py
│ │ │ │ ├── proxy
│ │ │ │ │ ├── examples.py
│ │ │ │ │ └── handlers.py
│ │ │ │ ├── responses.py
│ │ │ │ ├── tools
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── browser_tool.py
│ │ │ │ ├── types.py
│ │ │ │ └── ui
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __main__.py
│ │ │ │ └── gradio
│ │ │ │ ├── __init__.py
│ │ │ │ ├── app.py
│ │ │ │ └── ui_components.py
│ │ │ ├── benchmarks
│ │ │ │ ├── .gitignore
│ │ │ │ ├── contrib.md
│ │ │ │ ├── interactive.py
│ │ │ │ ├── models
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ └── gta1.py
│ │ │ │ ├── README.md
│ │ │ │ ├── ss-pro.py
│ │ │ │ ├── ss-v2.py
│ │ │ │ └── utils.py
│ │ │ ├── example.py
│ │ │ ├── pyproject.toml
│ │ │ ├── README.md
│ │ │ └── tests
│ │ │ ├── conftest.py
│ │ │ └── test_computer_agent.py
│ │ ├── bench-ui
│ │ │ ├── bench_ui
│ │ │ │ ├── __init__.py
│ │ │ │ ├── api.py
│ │ │ │ └── child.py
│ │ │ ├── examples
│ │ │ │ ├── folder_example.py
│ │ │ │ ├── gui
│ │ │ │ │ ├── index.html
│ │ │ │ │ ├── logo.svg
│ │ │ │ │ └── styles.css
│ │ │ │ ├── output_overlay.png
│ │ │ │ └── simple_example.py
│ │ │ ├── pyproject.toml
│ │ │ ├── README.md
│ │ │ └── tests
│ │ │ └── test_port_detection.py
│ │ ├── computer
│ │ │ ├── .bumpversion.cfg
│ │ │ ├── computer
│ │ │ │ ├── __init__.py
│ │ │ │ ├── computer.py
│ │ │ │ ├── diorama_computer.py
│ │ │ │ ├── helpers.py
│ │ │ │ ├── interface
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── factory.py
│ │ │ │ │ ├── generic.py
│ │ │ │ │ ├── linux.py
│ │ │ │ │ ├── macos.py
│ │ │ │ │ ├── models.py
│ │ │ │ │ └── windows.py
│ │ │ │ ├── logger.py
│ │ │ │ ├── models.py
│ │ │ │ ├── providers
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── cloud
│ │ │ │ │ │ ├── __init__.py
│ │ │ │ │ │ └── provider.py
│ │ │ │ │ ├── docker
│ │ │ │ │ │ ├── __init__.py
│ │ │ │ │ │ └── provider.py
│ │ │ │ │ ├── factory.py
│ │ │ │ │ ├── lume
│ │ │ │ │ │ ├── __init__.py
│ │ │ │ │ │ └── provider.py
│ │ │ │ │ ├── lume_api.py
│ │ │ │ │ ├── lumier
│ │ │ │ │ │ ├── __init__.py
│ │ │ │ │ │ └── provider.py
│ │ │ │ │ ├── types.py
│ │ │ │ │ └── winsandbox
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── provider.py
│ │ │ │ │ └── setup_script.ps1
│ │ │ │ ├── tracing_wrapper.py
│ │ │ │ ├── tracing.py
│ │ │ │ ├── ui
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── __main__.py
│ │ │ │ │ └── gradio
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── app.py
│ │ │ │ └── utils.py
│ │ │ ├── poetry.toml
│ │ │ ├── pyproject.toml
│ │ │ ├── README.md
│ │ │ └── tests
│ │ │ ├── conftest.py
│ │ │ └── test_computer.py
│ │ ├── computer-server
│ │ │ ├── .bumpversion.cfg
│ │ │ ├── computer_server
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __main__.py
│ │ │ │ ├── browser.py
│ │ │ │ ├── cli.py
│ │ │ │ ├── diorama
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── diorama_computer.py
│ │ │ │ │ ├── diorama.py
│ │ │ │ │ ├── draw.py
│ │ │ │ │ ├── macos.py
│ │ │ │ │ └── safezone.py
│ │ │ │ ├── handlers
│ │ │ │ │ ├── base.py
│ │ │ │ │ ├── factory.py
│ │ │ │ │ ├── generic.py
│ │ │ │ │ ├── linux.py
│ │ │ │ │ ├── macos.py
│ │ │ │ │ └── windows.py
│ │ │ │ ├── main.py
│ │ │ │ ├── server.py
│ │ │ │ ├── utils
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── wallpaper.py
│ │ │ │ └── watchdog.py
│ │ │ ├── examples
│ │ │ │ ├── __init__.py
│ │ │ │ └── usage_example.py
│ │ │ ├── pyproject.toml
│ │ │ ├── README.md
│ │ │ ├── run_server.py
│ │ │ ├── test_connection.py
│ │ │ └── tests
│ │ │ ├── conftest.py
│ │ │ └── test_server.py
│ │ ├── core
│ │ │ ├── .bumpversion.cfg
│ │ │ ├── core
│ │ │ │ ├── __init__.py
│ │ │ │ └── telemetry
│ │ │ │ ├── __init__.py
│ │ │ │ └── posthog.py
│ │ │ ├── poetry.toml
│ │ │ ├── pyproject.toml
│ │ │ ├── README.md
│ │ │ └── tests
│ │ │ ├── conftest.py
│ │ │ └── test_telemetry.py
│ │ ├── mcp-server
│ │ │ ├── .bumpversion.cfg
│ │ │ ├── build-extension.py
│ │ │ ├── CONCURRENT_SESSIONS.md
│ │ │ ├── desktop-extension
│ │ │ │ ├── cua-extension.mcpb
│ │ │ │ ├── desktop_extension.png
│ │ │ │ ├── manifest.json
│ │ │ │ ├── README.md
│ │ │ │ ├── requirements.txt
│ │ │ │ ├── run_server.sh
│ │ │ │ └── setup.py
│ │ │ ├── mcp_server
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __main__.py
│ │ │ │ ├── server.py
│ │ │ │ └── session_manager.py
│ │ │ ├── pdm.lock
│ │ │ ├── pyproject.toml
│ │ │ ├── QUICK_TEST_COMMANDS.sh
│ │ │ ├── quick_test_local_option.py
│ │ │ ├── README.md
│ │ │ ├── scripts
│ │ │ │ ├── install_mcp_server.sh
│ │ │ │ └── start_mcp_server.sh
│ │ │ ├── test_mcp_server_local_option.py
│ │ │ └── tests
│ │ │ ├── conftest.py
│ │ │ └── test_mcp_server.py
│ │ ├── pylume
│ │ │ └── tests
│ │ │ ├── conftest.py
│ │ │ └── test_pylume.py
│ │ └── som
│ │ ├── .bumpversion.cfg
│ │ ├── LICENSE
│ │ ├── poetry.toml
│ │ ├── pyproject.toml
│ │ ├── README.md
│ │ ├── som
│ │ │ ├── __init__.py
│ │ │ ├── detect.py
│ │ │ ├── detection.py
│ │ │ ├── models.py
│ │ │ ├── ocr.py
│ │ │ ├── util
│ │ │ │ └── utils.py
│ │ │ └── visualization.py
│ │ └── tests
│ │ ├── conftest.py
│ │ └── test_omniparser.py
│ ├── qemu-docker
│ │ ├── linux
│ │ │ ├── Dockerfile
│ │ │ ├── README.md
│ │ │ └── src
│ │ │ ├── entry.sh
│ │ │ └── vm
│ │ │ ├── image
│ │ │ │ └── README.md
│ │ │ └── setup
│ │ │ ├── install.sh
│ │ │ ├── setup-cua-server.sh
│ │ │ └── setup.sh
│ │ ├── README.md
│ │ └── windows
│ │ ├── Dockerfile
│ │ ├── README.md
│ │ └── src
│ │ ├── entry.sh
│ │ └── vm
│ │ ├── image
│ │ │ └── README.md
│ │ └── setup
│ │ ├── install.bat
│ │ ├── on-logon.ps1
│ │ ├── setup-cua-server.ps1
│ │ ├── setup-utils.psm1
│ │ └── setup.ps1
│ ├── typescript
│ │ ├── .gitignore
│ │ ├── .nvmrc
│ │ ├── agent
│ │ │ ├── examples
│ │ │ │ ├── playground-example.html
│ │ │ │ └── README.md
│ │ │ ├── package.json
│ │ │ ├── README.md
│ │ │ ├── src
│ │ │ │ ├── client.ts
│ │ │ │ ├── index.ts
│ │ │ │ └── types.ts
│ │ │ ├── tests
│ │ │ │ └── client.test.ts
│ │ │ ├── tsconfig.json
│ │ │ ├── tsdown.config.ts
│ │ │ └── vitest.config.ts
│ │ ├── computer
│ │ │ ├── .editorconfig
│ │ │ ├── .gitattributes
│ │ │ ├── .gitignore
│ │ │ ├── LICENSE
│ │ │ ├── package.json
│ │ │ ├── README.md
│ │ │ ├── src
│ │ │ │ ├── computer
│ │ │ │ │ ├── index.ts
│ │ │ │ │ ├── providers
│ │ │ │ │ │ ├── base.ts
│ │ │ │ │ │ ├── cloud.ts
│ │ │ │ │ │ └── index.ts
│ │ │ │ │ └── types.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── interface
│ │ │ │ │ ├── base.ts
│ │ │ │ │ ├── factory.ts
│ │ │ │ │ ├── index.ts
│ │ │ │ │ ├── linux.ts
│ │ │ │ │ ├── macos.ts
│ │ │ │ │ └── windows.ts
│ │ │ │ └── types.ts
│ │ │ ├── tests
│ │ │ │ ├── computer
│ │ │ │ │ └── cloud.test.ts
│ │ │ │ ├── interface
│ │ │ │ │ ├── factory.test.ts
│ │ │ │ │ ├── index.test.ts
│ │ │ │ │ ├── linux.test.ts
│ │ │ │ │ ├── macos.test.ts
│ │ │ │ │ └── windows.test.ts
│ │ │ │ └── setup.ts
│ │ │ ├── tsconfig.json
│ │ │ ├── tsdown.config.ts
│ │ │ └── vitest.config.ts
│ │ ├── core
│ │ │ ├── .editorconfig
│ │ │ ├── .gitattributes
│ │ │ ├── .gitignore
│ │ │ ├── LICENSE
│ │ │ ├── package.json
│ │ │ ├── README.md
│ │ │ ├── src
│ │ │ │ ├── index.ts
│ │ │ │ └── telemetry
│ │ │ │ ├── clients
│ │ │ │ │ ├── index.ts
│ │ │ │ │ └── posthog.ts
│ │ │ │ └── index.ts
│ │ │ ├── tests
│ │ │ │ └── telemetry.test.ts
│ │ │ ├── tsconfig.json
│ │ │ ├── tsdown.config.ts
│ │ │ └── vitest.config.ts
│ │ ├── cua-cli
│ │ │ ├── .gitignore
│ │ │ ├── .prettierrc
│ │ │ ├── bun.lock
│ │ │ ├── CLAUDE.md
│ │ │ ├── index.ts
│ │ │ ├── package.json
│ │ │ ├── README.md
│ │ │ ├── src
│ │ │ │ ├── auth.ts
│ │ │ │ ├── cli.ts
│ │ │ │ ├── commands
│ │ │ │ │ ├── auth.ts
│ │ │ │ │ └── sandbox.ts
│ │ │ │ ├── config.ts
│ │ │ │ ├── http.ts
│ │ │ │ ├── storage.ts
│ │ │ │ └── util.ts
│ │ │ └── tsconfig.json
│ │ ├── package.json
│ │ ├── pnpm-lock.yaml
│ │ ├── pnpm-workspace.yaml
│ │ └── README.md
│ └── xfce
│ ├── .dockerignore
│ ├── .gitignore
│ ├── Development.md
│ ├── Dockerfile
│ ├── Dockerfile.dev
│ ├── README.md
│ └── src
│ ├── scripts
│ │ ├── resize-display.sh
│ │ ├── start-computer-server.sh
│ │ ├── start-novnc.sh
│ │ ├── start-vnc.sh
│ │ └── xstartup.sh
│ ├── supervisor
│ │ └── supervisord.conf
│ └── xfce-config
│ ├── helpers.rc
│ ├── xfce4-power-manager.xml
│ └── xfce4-session.xml
├── LICENSE.md
├── Makefile
├── notebooks
│ ├── agent_nb.ipynb
│ ├── blog
│ │ ├── build-your-own-operator-on-macos-1.ipynb
│ │ └── build-your-own-operator-on-macos-2.ipynb
│ ├── composite_agents_docker_nb.ipynb
│ ├── computer_nb.ipynb
│ ├── computer_server_nb.ipynb
│ ├── customizing_computeragent.ipynb
│ ├── eval_osworld.ipynb
│ ├── ollama_nb.ipynb
│ ├── README.md
│ ├── sota_hackathon_cloud.ipynb
│ └── sota_hackathon.ipynb
├── package-lock.json
├── package.json
├── pnpm-lock.yaml
├── pyproject.toml
├── pyrightconfig.json
├── README.md
├── scripts
│ ├── install-cli.ps1
│ ├── install-cli.sh
│ ├── playground-docker.sh
│ ├── playground.sh
│ ├── run-docker-dev.sh
│ └── typescript-typecheck.js
├── TESTING.md
├── tests
│ ├── agent_loop_testing
│ │ ├── agent_test.py
│ │ └── README.md
│ ├── pytest.ini
│ ├── shell_cmd.py
│ ├── test_files.py
│ ├── test_mcp_server_session_management.py
│ ├── test_mcp_server_streaming.py
│ ├── test_shell_bash.py
│ ├── test_telemetry.py
│ ├── test_tracing.py
│ ├── test_venv.py
│ └── test_watchdog.py
└── uv.lock
```
# Files
--------------------------------------------------------------------------------
/.github/workflows/pypi-publish-som.yml:
--------------------------------------------------------------------------------
```yaml
1 | name: Publish SOM Package
2 |
3 | on:
4 | push:
5 | tags:
6 | - "som-v*"
7 | workflow_dispatch:
8 | inputs:
9 | version:
10 | description: "Version to publish (without v prefix)"
11 | required: true
12 | default: "0.1.0"
13 | workflow_call:
14 | inputs:
15 | version:
16 | description: "Version to publish"
17 | required: true
18 | type: string
19 | outputs:
20 | version:
21 | description: "The version that was published"
22 | value: ${{ jobs.determine-version.outputs.version }}
23 |
24 | # Adding permissions at workflow level
25 | permissions:
26 | contents: write
27 |
28 | jobs:
29 | determine-version:
30 | runs-on: macos-latest
31 | outputs:
32 | version: ${{ steps.get-version.outputs.version }}
33 | steps:
34 | - uses: actions/checkout@v4
35 |
36 | - name: Determine version
37 | id: get-version
38 | run: |
39 | if [ "${{ github.event_name }}" == "push" ]; then
40 | # Extract version from tag (for package-specific tags)
41 | if [[ "${{ github.ref }}" =~ ^refs/tags/som-v([0-9]+\.[0-9]+\.[0-9]+) ]]; then
42 | VERSION=${BASH_REMATCH[1]}
43 | else
44 | echo "Invalid tag format for som"
45 | exit 1
46 | fi
47 | elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
48 | # Use version from workflow dispatch
49 | VERSION=${{ github.event.inputs.version }}
50 | else
51 | # Use version from workflow_call
52 | VERSION=${{ inputs.version }}
53 | fi
54 | echo "VERSION=$VERSION"
55 | echo "version=$VERSION" >> $GITHUB_OUTPUT
56 |
57 | publish:
58 | needs: determine-version
59 | uses: ./.github/workflows/pypi-reusable-publish.yml
60 | with:
61 | package_name: "som"
62 | package_dir: "libs/python/som"
63 | version: ${{ needs.determine-version.outputs.version }}
64 | is_lume_package: false
65 | base_package_name: "cua-som"
66 | secrets:
67 | PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
68 |
```
--------------------------------------------------------------------------------
/docs/content/docs/agent-sdk/prompt-caching.mdx:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | title: Prompt Caching
3 | sidebar_position: 8
4 | description: How to use prompt caching in ComputerAgent and agent loops.
5 | ---
6 |
7 | Prompt caching is a cost-saving feature offered by some LLM API providers that helps avoid reprocessing the same prompt, improving efficiency and reducing costs for repeated or long-running tasks.
8 |
9 | ## Usage
10 |
11 | The `use_prompt_caching` argument is available for `ComputerAgent` and agent loops:
12 |
13 | ```python
14 | agent = ComputerAgent(
15 | ...,
16 | use_prompt_caching=True,
17 | )
18 | ```
19 |
20 | - **Type:** `bool`
21 | - **Default:** `False`
22 | - **Purpose:** Use prompt caching to avoid reprocessing the same prompt.
23 |
24 | ## Anthropic CUAs
25 |
26 | When using Anthropic-based CUAs (Claude models), setting `use_prompt_caching=True` will automatically add `{ "cache_control": "ephemeral" }` to your messages. This enables prompt caching for the session and can speed up repeated runs with the same prompt.
27 |
28 | <Callout title="Note">
29 | This argument is only required for Anthropic CUAs. For other providers, it is ignored.
30 | </Callout>
31 |
32 | ## OpenAI Provider
33 |
34 | With the OpenAI provider, prompt caching is handled automatically for prompts of 1000+ tokens. You do **not** need to set `use_prompt_caching`—caching will occur for long prompts without any extra configuration.
35 |
36 | ## Example
37 |
38 | ```python
39 | from agent import ComputerAgent
40 | agent = ComputerAgent(
41 | model="anthropic/claude-sonnet-4-5-20250929",
42 | use_prompt_caching=True,
43 | )
44 | ```
45 |
46 | ## Implementation Details
47 |
48 | - For Anthropic: Adds `{ "cache_control": "ephemeral" }` to messages when enabled.
49 | - For OpenAI: Caching is automatic for long prompts; the argument is ignored.
50 |
51 | ## When to Use
52 |
53 | - Enable for Anthropic CUAs if you want to avoid reprocessing the same prompt in repeated or iterative tasks.
54 | - Not needed for OpenAI models unless you want explicit ephemeral cache control (not required for most users).
55 |
56 | ## See Also
57 |
58 | - [Agent Loops](./agent-loops)
59 | - [Migration Guide](./migration-guide)
60 |
```
--------------------------------------------------------------------------------
/docs/content/docs/computer-sdk/computer-server/WebSocket-API.mdx:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | title: WebSocket API Reference
3 | description: Reference for the /ws WebSocket endpoint of the Computer Server.
4 | ---
5 |
6 | # WebSocket API Reference
7 |
8 | The Computer Server exposes a WebSocket endpoint for real-time command execution and streaming results.
9 |
10 | - `ws://localhost:8000/ws`
11 | - `wss://your-container.containers.cloud.trycua.com:8443/ws` (cloud)
12 |
13 | ### Authentication (Cloud Only)
14 |
15 | For cloud containers, you must authenticate immediately after connecting:
16 |
17 | ```json
18 | {
19 | "command": "authenticate",
20 | "params": {
21 | "container_name": "your-container",
22 | "api_key": "your-api-key"
23 | }
24 | }
25 | ```
26 |
27 | If authentication fails, the connection is closed.
28 |
29 | ### Command Format
30 |
31 | Send JSON messages:
32 |
33 | ```json
34 | {
35 | "command": "<command_name>",
36 | "params": { ... }
37 | }
38 | ```
39 |
40 | ### Example (Python)
41 |
42 | ```python
43 | import websockets
44 | import asyncio
45 | import json
46 |
47 | async def main():
48 | uri = "ws://localhost:8000/ws"
49 | async with websockets.connect(uri) as ws:
50 | await ws.send(json.dumps({"command": "version", "params": {}}))
51 | response = await ws.recv()
52 | print(response)
53 |
54 | asyncio.run(main())
55 | ```
56 |
57 | ### Example (Cloud)
58 |
59 | ```python
60 | import websockets
61 | import asyncio
62 | import json
63 |
64 | async def main():
65 | uri = "wss://your-container.containers.cloud.trycua.com:8443/ws"
66 | async with websockets.connect(uri) as ws:
67 | await ws.send(json.dumps({
68 | "command": "authenticate",
69 | "params": {
70 | "container_name": "your-container",
71 | "api_key": "your-api-key"
72 | }
73 | }))
74 | auth_response = await ws.recv()
75 | print(auth_response)
76 | await ws.send(json.dumps({"command": "version", "params": {}}))
77 | response = await ws.recv()
78 | print(response)
79 |
80 | asyncio.run(main())
81 | ```
82 |
83 | ### Response Format
84 |
85 | Each response is a JSON object:
86 |
87 | ```json
88 | {
89 | "success": true,
90 | ...
91 | }
92 | ```
93 |
94 | ### Supported Commands
95 |
96 | See [Commands Reference](./Commands) for the full list of commands and parameters.
97 |
```
--------------------------------------------------------------------------------
/docs/next.config.mjs:
--------------------------------------------------------------------------------
```
1 | import { createMDX } from 'fumadocs-mdx/next';
2 |
3 | const withMDX = createMDX();
4 |
5 | /** @type {import('next').NextConfig} */
6 | const config = {
7 | reactStrictMode: true,
8 | trailingSlash: false,
9 | basePath: '/docs',
10 | assetPrefix: '/docs',
11 | async rewrites() {
12 | return [
13 | {
14 | source: '/:path*.mdx',
15 | destination: '/llms.mdx/:path*',
16 | },
17 | ];
18 | },
19 | async redirects() {
20 | return [
21 | {
22 | source: '/',
23 | destination: '/docs',
24 | basePath: false, // Important: this bypasses the basePath
25 | permanent: false,
26 | },
27 | // Redirect old docs.cua.ai URLs to cua.ai/docs with 301 for SEO
28 | // This handles URLs that Google has indexed from the old domain
29 | {
30 | source: '/:path*',
31 | has: [
32 | {
33 | type: 'host',
34 | value: 'docs.cua.ai',
35 | },
36 | ],
37 | destination: 'https://cua.ai/docs/:path*',
38 | permanent: true, // 301 redirect to preserve SEO authority
39 | basePath: false,
40 | },
41 | // Redirects for documentation restructure (PR #568)
42 | // Moved quickstart-devs to get-started section
43 | {
44 | source: '/quickstart-devs',
45 | destination: '/get-started/quickstart',
46 | permanent: true,
47 | },
48 | // Moved telemetry to agent-sdk section
49 | {
50 | source: '/telemetry',
51 | destination: '/agent-sdk/telemetry',
52 | permanent: true,
53 | },
54 | // Removed quickstart-cli, consolidated into main quickstart
55 | {
56 | source: '/quickstart-cli',
57 | destination: '/get-started/quickstart',
58 | permanent: true,
59 | },
60 | ];
61 | },
62 | images: {
63 | dangerouslyAllowSVG: true,
64 | remotePatterns: [
65 | {
66 | protocol: 'https',
67 | hostname: 'img.shields.io',
68 | },
69 | {
70 | protocol: 'https',
71 | hostname: 'starchart.cc',
72 | },
73 | {
74 | protocol: 'https',
75 | hostname: 'github.com',
76 | },
77 | ],
78 | },
79 | };
80 |
81 | export default withMDX(config);
82 |
```
--------------------------------------------------------------------------------
/libs/python/computer/tests/conftest.py:
--------------------------------------------------------------------------------
```python
1 | """Pytest configuration and shared fixtures for computer package tests.
2 |
3 | This file contains shared fixtures and configuration for all computer tests.
4 | Following SRP: This file ONLY handles test setup/teardown.
5 | """
6 |
7 | from unittest.mock import AsyncMock, MagicMock, Mock, patch
8 |
9 | import pytest
10 |
11 |
12 | @pytest.fixture
13 | def mock_interface():
14 | """Mock computer interface for testing.
15 |
16 | Use this fixture to test Computer logic without real OS calls.
17 | """
18 | interface = AsyncMock()
19 | interface.screenshot = AsyncMock(return_value=b"fake_screenshot")
20 | interface.left_click = AsyncMock()
21 | interface.right_click = AsyncMock()
22 | interface.middle_click = AsyncMock()
23 | interface.double_click = AsyncMock()
24 | interface.type = AsyncMock()
25 | interface.key = AsyncMock()
26 | interface.move_mouse = AsyncMock()
27 | interface.scroll = AsyncMock()
28 | interface.get_screen_size = AsyncMock(return_value=(1920, 1080))
29 |
30 | return interface
31 |
32 |
33 | @pytest.fixture
34 | def mock_cloud_provider():
35 | """Mock cloud provider for testing.
36 |
37 | Use this fixture to test cloud provider logic without real API calls.
38 | """
39 | provider = AsyncMock()
40 | provider.start = AsyncMock()
41 | provider.stop = AsyncMock()
42 | provider.get_status = AsyncMock(return_value="running")
43 | provider.execute_command = AsyncMock(return_value="command output")
44 |
45 | return provider
46 |
47 |
48 | @pytest.fixture
49 | def mock_local_provider():
50 | """Mock local provider for testing.
51 |
52 | Use this fixture to test local provider logic without real VM operations.
53 | """
54 | provider = AsyncMock()
55 | provider.start = AsyncMock()
56 | provider.stop = AsyncMock()
57 | provider.get_status = AsyncMock(return_value="running")
58 | provider.execute_command = AsyncMock(return_value="command output")
59 |
60 | return provider
61 |
62 |
63 | @pytest.fixture
64 | def disable_telemetry(monkeypatch):
65 | """Disable telemetry for tests.
66 |
67 | Use this fixture to ensure no telemetry is sent during tests.
68 | """
69 | monkeypatch.setenv("CUA_TELEMETRY_DISABLED", "1")
70 |
```
--------------------------------------------------------------------------------
/libs/typescript/cua-cli/src/util.ts:
--------------------------------------------------------------------------------
```typescript
1 | export async function writeEnvFile(cwd: string, key: string) {
2 | const path = `${cwd}/.env`;
3 | let content = '';
4 | try {
5 | content = await Bun.file(path).text();
6 | } catch {}
7 | const lines = content.split(/\r?\n/).filter(Boolean);
8 | const idx = lines.findIndex((l) => l.startsWith('CUA_API_KEY='));
9 | if (idx >= 0) lines[idx] = `CUA_API_KEY=${key}`;
10 | else lines.push(`CUA_API_KEY=${key}`);
11 | await Bun.write(path, lines.join('\n') + '\n');
12 | return path;
13 | }
14 |
15 | export type SandboxStatus =
16 | | 'pending'
17 | | 'running'
18 | | 'stopped'
19 | | 'suspended'
20 | | 'suspending'
21 | | 'terminated'
22 | | 'failed';
23 | export type SandboxItem = {
24 | name: string;
25 | password: string;
26 | status: SandboxStatus;
27 | host?: string;
28 | };
29 |
30 | export function printSandboxList(
31 | items: SandboxItem[],
32 | showPasswords: boolean = false
33 | ) {
34 | const headers = showPasswords
35 | ? ['NAME', 'STATUS', 'PASSWORD', 'HOST']
36 | : ['NAME', 'STATUS', 'HOST'];
37 |
38 | const rows: string[][] = [
39 | headers,
40 | ...items.map((v) =>
41 | showPasswords
42 | ? [v.name, String(v.status), v.password, v.host || '']
43 | : [v.name, String(v.status), v.host || '']
44 | ),
45 | ];
46 |
47 | const numCols = headers.length;
48 | const widths: number[] = new Array(numCols).fill(0);
49 |
50 | for (const r of rows)
51 | for (let i = 0; i < numCols; i++)
52 | widths[i] = Math.max(widths[i] ?? 0, (r[i] ?? '').length);
53 |
54 | for (const r of rows)
55 | console.log(r.map((c, i) => (c ?? '').padEnd(widths[i] ?? 0)).join(' '));
56 |
57 | if (items.length === 0) console.log('No sandboxes found');
58 | }
59 |
60 | export async function openInBrowser(url: string) {
61 | const platform = process.platform;
62 | let cmd: string;
63 | let args: string[] = [];
64 | if (platform === 'darwin') {
65 | cmd = 'open';
66 | args = [url];
67 | } else if (platform === 'win32') {
68 | cmd = 'cmd';
69 | args = ['/c', 'start', '', url];
70 | } else {
71 | cmd = 'xdg-open';
72 | args = [url];
73 | }
74 | try {
75 | await Bun.spawn({ cmd: [cmd, ...args] }).exited;
76 | } catch {
77 | console.error(`Failed to open browser. Please visit: ${url}`);
78 | }
79 | }
80 |
```
--------------------------------------------------------------------------------
/docs/content/docs/agent-sdk/integrations/observability.mdx:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | title: Observability
3 | description: Trace CUA execution steps and sessions
4 | ---
5 |
6 | ## Observability
7 |
8 | CUA has a native integration with [Laminar](https://laminar.sh/) – open-source platform for tracing, evals, and labeling of autonomous AI agents. Read more about Laminar in the [Laminar docs](https://docs.lmnr.ai/).
9 |
10 | ## Setup
11 |
12 | Register on [Laminar Cloud](https://laminar.sh/) or spin up a [local instance](https://github.com/lmnr-ai/lmnr) and get the key from your project settings. Set the `LMNR_PROJECT_API_KEY` environment variable to your key.
13 |
14 | ```bash
15 | pip install lmnr[all]
16 | export LMNR_PROJECT_API_KEY=your-key
17 | ```
18 |
19 | ## Usage
20 |
21 | Then, initialize Laminar at the entry point of your application, register Laminar LiteLLM callback, and all steps of CUA will be automatically traced.
22 |
23 | ```python
24 | import os
25 |
26 | import litellm
27 |
28 | from agent import ComputerAgent
29 | from computer import Computer
30 | from lmnr import Laminar, LaminarLiteLLMCallback # [!code highlight]
31 |
32 | Laminar.initialize() # [!code highlight]
33 | litellm.callbacks.append(LaminarLiteLLMCallback()) # [!code highlight]
34 |
35 | computer = Computer(
36 | os_type="linux",
37 | provider_type="cloud",
38 | name=os.getenv("CUA_CONTAINER_NAME"),
39 | api_key=os.getenv("CUA_API_KEY"),
40 | )
41 |
42 | agent = ComputerAgent(
43 | model="openai/computer-use-preview",
44 | tools=[computer],
45 | )
46 |
47 | async def main():
48 | async for step in agent.run("Create a new file called 'test.txt' in the current directory"):
49 | print(step["output"])
50 |
51 | if __name__ == "__main__":
52 | asyncio.run(main())
53 | ```
54 |
55 | ## Viewing traces
56 |
57 | You can view traces in the Laminar UI by going to the traces tab in your project. When you select a trace,
58 | you will see all the agent execution steps, including computer actions, LLM calls, and screenshots.
59 |
60 | For each step, you will see the LLM call, the computer action. The computer actions are highlighted in the timeline in yellow.
61 |
62 | <img
63 | src="/docs/img/laminar_trace_example.png"
64 | alt="Example trace in Laminar showing the litellm.response span and its output."
65 | width="800px"
66 | />
67 |
```
--------------------------------------------------------------------------------
/libs/python/computer-server/examples/usage_example.py:
--------------------------------------------------------------------------------
```python
1 | #!/usr/bin/env python3
2 | """
3 | Example showing how to use the CUA Computer API as an imported package.
4 | """
5 |
6 | import asyncio
7 | import logging
8 | from typing import TYPE_CHECKING
9 |
10 | # For type checking only
11 | if TYPE_CHECKING:
12 | from computer_api import Server
13 |
14 | # Setup logging
15 | logging.basicConfig(
16 | level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
17 | )
18 | logger = logging.getLogger(__name__)
19 |
20 |
21 | # Example 1: Synchronous usage (blocks until server is stopped)
22 | def example_sync():
23 | """
24 | Example of synchronous server usage. This will block until interrupted.
25 | Run with: python3 -m examples.usage_example sync
26 | """
27 | # Import directly to avoid any confusion
28 | from computer_api.server import Server
29 |
30 | server = Server(port=8080)
31 | print("Server started at http://localhost:8080")
32 | print("Press Ctrl+C to stop the server")
33 |
34 | try:
35 | server.start() # This will block until the server is stopped
36 | except KeyboardInterrupt:
37 | print("Server stopped by user")
38 |
39 |
40 | # Example 2: Asynchronous usage
41 | async def example_async():
42 | """
43 | Example of asynchronous server usage. This will start the server in the background
44 | and allow other operations to run concurrently.
45 | Run with: python3 -m examples.usage_example async
46 | """
47 | # Import directly to avoid any confusion
48 | from computer_api.server import Server
49 |
50 | server = Server(port=8080)
51 |
52 | # Start the server in the background
53 | await server.start_async()
54 |
55 | print("Server is running in the background")
56 | print("Performing other tasks...")
57 |
58 | # Do other things while the server is running
59 | for i in range(5):
60 | print(f"Doing work iteration {i+1}/5...")
61 | await asyncio.sleep(2)
62 |
63 | print("Work complete, stopping server...")
64 |
65 | # Stop the server when done
66 | await server.stop()
67 | print("Server stopped")
68 |
69 |
70 | if __name__ == "__main__":
71 | import sys
72 |
73 | if len(sys.argv) > 1 and sys.argv[1] == "async":
74 | asyncio.run(example_async())
75 | else:
76 | example_sync()
77 |
```
--------------------------------------------------------------------------------
/libs/lume/tests/VMVirtualizationServiceTests.swift:
--------------------------------------------------------------------------------
```swift
1 | import Foundation
2 | import Testing
3 | import Virtualization
4 | @testable import lume
5 |
6 | @Test("VMVirtualizationService starts correctly")
7 | func testVMVirtualizationServiceStart() async throws {
8 | let service = MockVMVirtualizationService()
9 |
10 | // Initial state
11 | #expect(await service.state == .stopped)
12 | #expect(await service.startCallCount == 0)
13 |
14 | // Start service
15 | try await service.start()
16 | #expect(await service.state == .running)
17 | #expect(await service.startCallCount == 1)
18 | }
19 |
20 | @Test("VMVirtualizationService stops correctly")
21 | func testVMVirtualizationServiceStop() async throws {
22 | let service = MockVMVirtualizationService()
23 |
24 | // Start then stop
25 | try await service.start()
26 | try await service.stop()
27 |
28 | #expect(await service.state == .stopped)
29 | #expect(await service.stopCallCount == 1)
30 | }
31 |
32 | @Test("VMVirtualizationService handles pause and resume")
33 | func testVMVirtualizationServicePauseResume() async throws {
34 | let service = MockVMVirtualizationService()
35 |
36 | // Start and pause
37 | try await service.start()
38 | try await service.pause()
39 | #expect(await service.state == .paused)
40 | #expect(await service.pauseCallCount == 1)
41 |
42 | // Resume
43 | try await service.resume()
44 | #expect(await service.state == .running)
45 | #expect(await service.resumeCallCount == 1)
46 | }
47 |
48 | @Test("VMVirtualizationService handles operation failures")
49 | func testVMVirtualizationServiceFailures() async throws {
50 | let service = MockVMVirtualizationService()
51 | await service.configure(shouldFail: true)
52 |
53 | // Test start failure
54 | do {
55 | try await service.start()
56 | #expect(Bool(false), "Expected start to throw")
57 | } catch let error as VMError {
58 | switch error {
59 | case .internalError(let message):
60 | #expect(message == "Mock operation failed")
61 | default:
62 | #expect(Bool(false), "Unexpected error type: \(error)")
63 | }
64 | }
65 |
66 | #expect(await service.state == .stopped)
67 | #expect(await service.startCallCount == 1)
68 | }
```
--------------------------------------------------------------------------------
/libs/lumier/src/hooks/on-logon.sh:
--------------------------------------------------------------------------------
```bash
1 | #!/bin/bash
2 |
3 | # Arguments passed from execute_remote_script in vm.sh
4 | # $1: VNC_PASSWORD
5 | # $2: HOST_SHARED_PATH (Path inside VM where host shared dir is mounted, e.g., /Volumes/My Shared Files)
6 |
7 | VNC_PASSWORD="$1"
8 | # IMPORTANT: In the VM, the shared folder is always mounted at this fixed location
9 | HOST_SHARED_PATH="/Volumes/My Shared Files"
10 |
11 | # Set default value for VNC_DEBUG if not provided
12 | VNC_DEBUG=${VNC_DEBUG:-0}
13 |
14 | # Define the path to the user's optional on-logon script within the shared folder
15 | USER_ON_LOGON_SCRIPT_PATH="$HOST_SHARED_PATH/lifecycle/on-logon.sh"
16 |
17 | # Show basic information when debug is enabled
18 | if [ "$VNC_DEBUG" = "1" ]; then
19 | echo "[VM] Lumier lifecycle script starting"
20 | echo "[VM] Looking for user script: $USER_ON_LOGON_SCRIPT_PATH"
21 | fi
22 |
23 | # Check if the user-provided script exists
24 | if [ -f "$USER_ON_LOGON_SCRIPT_PATH" ]; then
25 | if [ "$VNC_DEBUG" = "1" ]; then
26 | echo "[VM] Found user script: $USER_ON_LOGON_SCRIPT_PATH"
27 | fi
28 |
29 | # Always show what script we're executing
30 | echo "[VM] Executing user lifecycle script"
31 |
32 | # Make script executable
33 | chmod +x "$USER_ON_LOGON_SCRIPT_PATH"
34 |
35 | # Execute the user script in a subshell with error output captured
36 | "$USER_ON_LOGON_SCRIPT_PATH" "$VNC_PASSWORD" "$HOST_SHARED_PATH" 2>&1
37 |
38 | # Capture exit code
39 | USER_SCRIPT_EXIT_CODE=$?
40 |
41 | # Always report script execution results
42 | if [ $USER_SCRIPT_EXIT_CODE -eq 0 ]; then
43 | echo "[VM] User lifecycle script completed successfully"
44 | else
45 | echo "[VM] User lifecycle script failed with exit code: $USER_SCRIPT_EXIT_CODE"
46 | fi
47 |
48 | # Check results (only in debug mode)
49 | if [ "$VNC_DEBUG" = "1" ]; then
50 | # List any files created by the script
51 | echo "[VM] Files created by user script:"
52 | ls -la /Users/lume/Desktop/hello_*.txt 2>/dev/null || echo "[VM] No script-created files found"
53 | fi
54 | else
55 | if [ "$VNC_DEBUG" = "1" ]; then
56 | echo "[VM] No user lifecycle script found"
57 | fi
58 | fi
59 |
60 | exit 0 # Ensure the entry point script exits cleanly
61 |
```
--------------------------------------------------------------------------------
/examples/computer-example-ts/src/helpers.ts:
--------------------------------------------------------------------------------
```typescript
1 | import type { Computer } from '@trycua/computer';
2 | import type OpenAI from 'openai';
3 |
4 | export async function executeAction(
5 | computer: Computer,
6 | action: OpenAI.Responses.ResponseComputerToolCall['action']
7 | ) {
8 | switch (action.type) {
9 | case 'click': {
10 | const { x, y, button } = action;
11 | console.log(`Executing click at (${x}, ${y}) with button '${button}'.`);
12 | await computer.interface.moveCursor(x, y);
13 | if (button === 'right') await computer.interface.rightClick();
14 | else await computer.interface.leftClick();
15 | break;
16 | }
17 | case 'type':
18 | {
19 | const { text } = action;
20 | console.log(`Typing text: ${text}`);
21 | await computer.interface.typeText(text);
22 | }
23 | break;
24 | case 'scroll': {
25 | const { x: locX, y: locY, scroll_x, scroll_y } = action;
26 | console.log(
27 | `Scrolling at (${locX}, ${locY}) with offsets (scroll_x=${scroll_x}, scroll_y=${scroll_y}).`
28 | );
29 | await computer.interface.moveCursor(locX, locY);
30 | await computer.interface.scroll(scroll_x, scroll_y);
31 | break;
32 | }
33 | case 'keypress': {
34 | const { keys } = action;
35 | for (const key of keys) {
36 | console.log(`Pressing key: ${key}.`);
37 | // Map common key names to CUA equivalents
38 | if (key.toLowerCase() === 'enter') {
39 | await computer.interface.pressKey('return');
40 | } else if (key.toLowerCase() === 'space') {
41 | await computer.interface.pressKey('space');
42 | } else {
43 | await computer.interface.pressKey(key);
44 | }
45 | }
46 | break;
47 | }
48 | case 'wait': {
49 | console.log(`Waiting for 3 seconds.`);
50 | await new Promise((resolve) => setTimeout(resolve, 3 * 1000));
51 | break;
52 | }
53 | case 'screenshot': {
54 | console.log('Taking screenshot.');
55 | // This is handled automatically in the main loop, but we can take an extra one if requested
56 | const screenshot = await computer.interface.screenshot();
57 | return screenshot;
58 | }
59 | default:
60 | console.log(`Unrecognized action: ${action.type}`);
61 | break;
62 | }
63 | }
64 |
```
--------------------------------------------------------------------------------
/docs/content/docs/agent-sdk/callbacks/cost-saving.mdx:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | title: Cost Optimization
3 | description: Budget management and image retention for cost optimization
4 | ---
5 |
6 | # Cost Optimization Callbacks
7 |
8 | Optimize agent costs with budget management and image retention callbacks.
9 |
10 | ## Budget Manager Callbacks Example
11 |
12 | ```python
13 | from agent.callbacks import BudgetManagerCallback
14 |
15 | agent = ComputerAgent(
16 | model="anthropic/claude-sonnet-4-5-20250929",
17 | tools=[computer],
18 | callbacks=[
19 | BudgetManagerCallback(
20 | max_budget=5.0, # $5 limit
21 | reset_after_each_run=False,
22 | raise_error=True
23 | )
24 | ]
25 | )
26 | ```
27 |
28 | ## Budget Manager Shorthand
29 |
30 | ```python
31 | # Simple budget limit
32 | agent = ComputerAgent(
33 | model="anthropic/claude-sonnet-4-5-20250929",
34 | max_trajectory_budget=5.0 # $5 limit
35 | )
36 | ```
37 |
38 | **Or with options:**
39 |
40 | ```python
41 | # Advanced budget configuration
42 | agent = ComputerAgent(
43 | model="anthropic/claude-sonnet-4-5-20250929",
44 | max_trajectory_budget={
45 | "max_budget": 10.0,
46 | "raise_error": True, # Raise error when exceeded
47 | "reset_after_each_run": False # Persistent across runs
48 | }
49 | )
50 | ```
51 |
52 | ## Image Retention Callbacks Example
53 |
54 | ```python
55 | from agent.callbacks import ImageRetentionCallback
56 |
57 | agent = ComputerAgent(
58 | model="anthropic/claude-sonnet-4-5-20250929",
59 | tools=[computer],
60 | callbacks=[
61 | ImageRetentionCallback(only_n_most_recent_images=3)
62 | ]
63 | )
64 | ```
65 |
66 | ## Image Retention Shorthand
67 |
68 | ```python
69 | agent = ComputerAgent(
70 | model="anthropic/claude-sonnet-4-5-20250929",
71 | tools=[computer],
72 | only_n_most_recent_images=3 # Auto-adds ImageRetentionCallback
73 | )
74 | ```
75 |
76 | ## Combined Cost Optimization
77 |
78 | ```python
79 | agent = ComputerAgent(
80 | model="anthropic/claude-sonnet-4-5-20250929",
81 | tools=[computer],
82 | max_trajectory_budget=5.0, # Budget limit
83 | only_n_most_recent_images=3, # Image retention
84 | trajectory_dir="trajectories" # Track spending
85 | )
86 | ```
87 |
88 | ## Budget Manager Options
89 |
90 | - `max_budget`: Dollar limit for trajectory
91 | - `reset_after_each_run`: Reset budget per run (default: True)
92 | - `raise_error`: Raise exception vs. graceful stop (default: False)
93 |
```
--------------------------------------------------------------------------------
/docs/content/docs/agent-sdk/callbacks/index.mdx:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | title: Callbacks
3 | ---
4 |
5 | Callbacks in the Agent SDK provide hooks into the agent's lifecycle, allowing for custom functionality to be executed at various stages of an agent's run. They enable extensibility by allowing developers to integrate their own logic for tasks such as logging, cost management, and data anonymization.
6 |
7 | ## Usage
8 |
9 | You can add preprocessing and postprocessing hooks using callbacks, or write your own by subclassing `AsyncCallbackHandler`.
10 |
11 | ### Built-in Callbacks
12 |
13 | Built-in callbacks can be used as follows:
14 |
15 | ```python
16 | from agent.callbacks import (
17 | ImageRetentionCallback,
18 | TrajectorySaverCallback,
19 | BudgetManagerCallback,
20 | LoggingCallback
21 | )
22 |
23 | agent = ComputerAgent(
24 | model="anthropic/claude-sonnet-4-5-20250929",
25 | tools=[computer],
26 | callbacks=[
27 | ImageRetentionCallback(only_n_most_recent_images=3),
28 | TrajectorySaverCallback(trajectory_dir="trajectories"),
29 | BudgetManagerCallback(max_budget=10.0, raise_error=True),
30 | LoggingCallback(level=logging.INFO)
31 | ]
32 | )
33 | ```
34 |
35 | The following built-in callbacks are available:
36 |
37 | - [BudgetManagerCallback](callbacks/cost-saving): Stops execution when budget exceeded
38 | - [LoggingCallback](callbacks/trajectories): Logs agent activities
39 | - **ImageRetentionCallback**: Limits recent images in context
40 | - **TrajectorySaverCallback**: Saves conversation trajectories
41 | - [PII Anonymization](callbacks/pii-anonymization)
42 |
43 | ### Custom Callbacks
44 |
45 | Create custom callbacks using knowlege of the callback lifecycle as described in [Agent Lifecycle](callbacks/agent-lifecycle).
46 |
47 | ```python
48 | from agent.callbacks.base import AsyncCallbackHandler
49 |
50 | class CustomCallback(AsyncCallbackHandler):
51 | async def on_llm_start(self, messages):
52 | """Preprocess messages before LLM call"""
53 | # Add custom preprocessing logic
54 | return messages
55 |
56 | async def on_llm_end(self, messages):
57 | """Postprocess messages after LLM call"""
58 | # Add custom postprocessing logic
59 | return messages
60 |
61 | async def on_usage(self, usage):
62 | """Track usage information"""
63 | print(f"Tokens used: {usage.total_tokens}")
64 | ```
65 |
```
--------------------------------------------------------------------------------
/docs/content/docs/macos-vm-cli-playbook/lume/prebuilt-images.mdx:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | title: Prebuilt Images
3 | ---
4 |
5 | Pre-built images are available in the registry [ghcr.io/trycua](https://github.com/orgs/trycua/packages). These images come with an SSH server pre-configured and auto-login enabled.
6 |
7 | <Callout>
8 | The default password on pre-built images is `lume`. For the security of your VM, change this
9 | password after your first login.
10 | </Callout>
11 |
12 | ## Available Images
13 |
14 | The following pre-built images are available to download via `lume pull`:
15 |
16 | | Image | Tag | Description | Logical Size |
17 | | ----------------------- | ------------------- | ----------------------------------------------------------------------------------------------- | ------------ |
18 | | `macos-sequoia-vanilla` | `latest`, `15.2` | macOS Sequoia 15.2 image | 20GB |
19 | | `macos-sequoia-xcode` | `latest`, `15.2` | macOS Sequoia 15.2 image with Xcode command line tools | 22GB |
20 | | `macos-sequoia-cua` | `latest`, `15.3` | macOS Sequoia 15.3 image compatible with the Computer interface | 24GB |
21 | | `ubuntu-noble-vanilla` | `latest`, `24.04.1` | [Ubuntu Server for ARM 24.04.1 LTS](https://ubuntu.com/download/server/arm) with Ubuntu Desktop | 20GB |
22 |
23 | ## Disk Space
24 |
25 | For additional disk space, resize the VM disk after pulling the image using the `lume set <name> --disk-size <size>` command. Note that the actual disk space used by sparse images will be much lower than the logical size listed.
26 |
27 | <Callout>
28 | **Important Note (v0.2.0+):** Images are being re-uploaded with sparse file system optimizations
29 | enabled, resulting in significantly lower actual disk usage. Older images (without the `-sparse`
30 | suffix) are now **deprecated**. The last version of `lume` fully supporting the non-sparse images
31 | was `v0.1.x`. Starting from `v0.2.0`, lume will automatically pull images optimized with sparse
32 | file system support.
33 | </Callout>
34 |
```
--------------------------------------------------------------------------------
/docs/src/app/api/posthog/[...path]/route.ts:
--------------------------------------------------------------------------------
```typescript
1 | import { NextRequest, NextResponse } from 'next/server';
2 |
3 | export async function GET(
4 | request: NextRequest,
5 | { params }: { params: Promise<{ path: string[] }> }
6 | ) {
7 | const { path } = await params;
8 | const url = new URL(request.url);
9 |
10 | const targetUrl = `${process.env.NEXT_PUBLIC_POSTHOG_HOST}/${path.join('/')}${url.search}`;
11 |
12 | try {
13 | const response = await fetch(targetUrl, {
14 | method: 'GET',
15 | headers: {
16 | 'Content-Type': request.headers.get('Content-Type') || 'application/json',
17 | },
18 | });
19 |
20 | // Handle 204 No Content responses
21 | if (response.status === 204) {
22 | return new NextResponse(null, { status: 204 });
23 | }
24 |
25 | const data = await response.arrayBuffer();
26 | return new NextResponse(data, {
27 | status: response.status,
28 | headers: {
29 | 'Content-Type': response.headers.get('Content-Type') || 'application/json',
30 | },
31 | });
32 | } catch (error) {
33 | console.error('PostHog proxy error:', error);
34 | return new NextResponse('Error proxying request', { status: 500 });
35 | }
36 | }
37 |
38 | export async function POST(
39 | request: NextRequest,
40 | { params }: { params: Promise<{ path: string[] }> }
41 | ) {
42 | const { path } = await params;
43 | const url = new URL(request.url);
44 |
45 | const targetUrl = `${process.env.NEXT_PUBLIC_POSTHOG_HOST}/${path.join('/')}${url.search}`;
46 |
47 | try {
48 | const body = await request.arrayBuffer();
49 | const contentType = request.headers.get('Content-Type') || 'application/x-www-form-urlencoded';
50 |
51 | const response = await fetch(targetUrl, {
52 | method: 'POST',
53 | headers: {
54 | 'Content-Type': contentType,
55 | },
56 | body,
57 | });
58 |
59 | // Handle 204 No Content responses
60 | if (response.status === 204) {
61 | return new NextResponse(null, { status: 204 });
62 | }
63 |
64 | const data = await response.arrayBuffer();
65 | return new NextResponse(data, {
66 | status: response.status,
67 | headers: {
68 | 'Content-Type': response.headers.get('Content-Type') || 'application/json',
69 | },
70 | });
71 | } catch (error) {
72 | console.error('PostHog proxy error:', error);
73 | return new NextResponse('Error proxying request', { status: 500 });
74 | }
75 | }
76 |
```
--------------------------------------------------------------------------------
/libs/typescript/computer/tests/interface/factory.test.ts:
--------------------------------------------------------------------------------
```typescript
1 | import { describe, expect, it } from 'vitest';
2 | import { InterfaceFactory } from '../../src/interface/factory.ts';
3 | import { LinuxComputerInterface } from '../../src/interface/linux.ts';
4 | import { MacOSComputerInterface } from '../../src/interface/macos.ts';
5 | import { WindowsComputerInterface } from '../../src/interface/windows.ts';
6 | import { OSType } from '../../src/types.ts';
7 |
8 | describe('InterfaceFactory', () => {
9 | const testParams = {
10 | ipAddress: '192.168.1.100',
11 | username: 'testuser',
12 | password: 'testpass',
13 | apiKey: 'test-api-key',
14 | vmName: 'test-vm',
15 | };
16 |
17 | describe('createInterfaceForOS', () => {
18 | it('should create MacOSComputerInterface for macOS', () => {
19 | const interface_ = InterfaceFactory.createInterfaceForOS(
20 | OSType.MACOS,
21 | testParams.ipAddress,
22 | testParams.apiKey,
23 | testParams.vmName
24 | );
25 |
26 | expect(interface_).toBeInstanceOf(MacOSComputerInterface);
27 | });
28 |
29 | it('should create LinuxComputerInterface for Linux', () => {
30 | const interface_ = InterfaceFactory.createInterfaceForOS(
31 | OSType.LINUX,
32 | testParams.ipAddress,
33 | testParams.apiKey,
34 | testParams.vmName
35 | );
36 |
37 | expect(interface_).toBeInstanceOf(LinuxComputerInterface);
38 | });
39 |
40 | it('should create WindowsComputerInterface for Windows', () => {
41 | const interface_ = InterfaceFactory.createInterfaceForOS(
42 | OSType.WINDOWS,
43 | testParams.ipAddress,
44 | testParams.apiKey,
45 | testParams.vmName
46 | );
47 |
48 | expect(interface_).toBeInstanceOf(WindowsComputerInterface);
49 | });
50 |
51 | it('should throw error for unsupported OS type', () => {
52 | expect(() => {
53 | InterfaceFactory.createInterfaceForOS(
54 | 'unsupported' as OSType,
55 | testParams.ipAddress,
56 | testParams.apiKey,
57 | testParams.vmName
58 | );
59 | }).toThrow('Unsupported OS type: unsupported');
60 | });
61 |
62 | it('should create interface without API key and VM name', () => {
63 | const interface_ = InterfaceFactory.createInterfaceForOS(OSType.MACOS, testParams.ipAddress);
64 |
65 | expect(interface_).toBeInstanceOf(MacOSComputerInterface);
66 | });
67 | });
68 | });
69 |
```
--------------------------------------------------------------------------------
/libs/typescript/cua-cli/src/cli.ts:
--------------------------------------------------------------------------------
```typescript
1 | import yargs from 'yargs';
2 | import { hideBin } from 'yargs/helpers';
3 | import { registerAuthCommands } from './commands/auth';
4 | import { registerSandboxCommands } from './commands/sandbox';
5 |
6 | export async function runCli() {
7 | let argv = yargs(hideBin(process.argv))
8 | .scriptName('cua')
9 | .usage('Usage: $0 <command> [options]')
10 | .epilogue(
11 | 'Recommended Command Structure:\n' +
12 | ' cua auth <command> Authenticate and manage credentials\n' +
13 | ' login Login via browser or with API key\n' +
14 | ' env Export API key to .env file\n' +
15 | ' logout Clear stored credentials\n' +
16 | '\n' +
17 | ' cua sb <command> Create and manage cloud sandboxes\n' +
18 | ' list View all your sandboxes\n' +
19 | ' create Provision a new sandbox\n' +
20 | ' get Get detailed info about a sandbox\n' +
21 | ' start Start or resume a sandbox\n' +
22 | ' stop Stop a sandbox (preserves disk)\n' +
23 | ' suspend Suspend a sandbox (preserves memory)\n' +
24 | ' vnc Open remote desktop\n' +
25 | '\n' +
26 | 'Documentation: https://docs.cua.ai/libraries/cua-cli/commands'
27 | );
28 | // Override the default --version behavior
29 | argv = argv.version(false).option('version', {
30 | alias: 'v',
31 | describe: 'Show CUA CLI version',
32 | type: 'boolean',
33 | global: false,
34 | });
35 | argv = registerAuthCommands(argv);
36 | argv = registerSandboxCommands(argv);
37 |
38 | // Check for version flag before command validation
39 | const args = process.argv.slice(2);
40 | if (args.includes('--version') || args.includes('-v')) {
41 | try {
42 | const home = process.env.HOME || process.env.USERPROFILE || '';
43 | const path = `${home}/.cua/bin/.version`;
44 | const version = await Bun.file(path).text();
45 | const v = version.trim();
46 | if (v) {
47 | console.log(v);
48 | } else {
49 | console.log('unknown');
50 | }
51 | } catch {
52 | console.log('unknown');
53 | }
54 | process.exit(0);
55 | }
56 |
57 | await argv.demandCommand(1).strict().help().parseAsync();
58 | }
59 |
```
--------------------------------------------------------------------------------
/libs/python/agent/pyproject.toml:
--------------------------------------------------------------------------------
```toml
1 | [build-system]
2 | requires = ["pdm-backend"]
3 | build-backend = "pdm.backend"
4 |
5 | [project]
6 | name = "cua-agent"
7 | version = "0.5.2"
8 | description = "CUA (Computer Use) Agent for AI-driven computer interaction"
9 | readme = "README.md"
10 | authors = [
11 | { name = "TryCua", email = "[email protected]" }
12 | ]
13 | dependencies = [
14 | "httpx>=0.27.0",
15 | "aiohttp>=3.9.3",
16 | "asyncio",
17 | "anyio>=4.4.1",
18 | "typing-extensions>=4.12.2",
19 | "pydantic>=2.6.4",
20 | "rich>=13.7.1",
21 | "python-dotenv>=1.0.1",
22 | "cua-computer>=0.4.0,<0.5.0",
23 | "cua-core>=0.1.8,<0.2.0",
24 | "certifi>=2024.2.2",
25 | "litellm>=1.74.12"
26 | ]
27 | requires-python = ">=3.12,<3.14"
28 |
29 | [project.optional-dependencies]
30 | openai = []
31 | anthropic = []
32 | qwen = [
33 | "qwen-vl-utils",
34 | "qwen-agent",
35 | "Pillow>=10.0.0",
36 | ]
37 | omni = [
38 | "cua-som>=0.1.0,<0.2.0",
39 | ]
40 | uitars = []
41 | uitars-mlx = [
42 | "mlx-vlm>=0.1.27; sys_platform == 'darwin'"
43 | ]
44 | uitars-hf = [
45 | "accelerate",
46 | "torch",
47 | "transformers>=4.54.0"
48 | ]
49 | glm45v-hf = [
50 | "accelerate",
51 | "torch",
52 | "transformers-v4.55.0-GLM-4.5V-preview"
53 | ]
54 | opencua-hf = [
55 | "accelerate",
56 | "torch",
57 | "transformers>=4.53.0",
58 | "tiktoken>=0.11.0",
59 | "blobfile>=3.0.0"
60 | ]
61 | internvl-hf = [
62 | "accelerate",
63 | "torch",
64 | "transformers>=4.55.0",
65 | "einops",
66 | "timm"
67 | ]
68 | moondream3 = [
69 | "accelerate",
70 | "torch",
71 | "transformers>=4.55.0"
72 | ]
73 | ui = [
74 | "gradio>=5.23.3",
75 | "python-dotenv>=1.0.1",
76 | ]
77 | cli = [
78 | "yaspin>=3.1.0",
79 | ]
80 | hud = [
81 | "hud-python==0.4.52",
82 | ]
83 | gemini = [
84 | "google-genai>=1.41.0",
85 | ]
86 | all = [
87 | # uitars requirements
88 | "mlx-vlm>=0.1.27; sys_platform == 'darwin'",
89 | "accelerate",
90 | "torch",
91 | "transformers>=4.55.0",
92 | # internvl requirements,
93 | "einops",
94 | "timm",
95 | # opencua requirements
96 | "tiktoken>=0.11.0",
97 | "blobfile>=3.0.0",
98 | # ui requirements
99 | "gradio>=5.23.3",
100 | "python-dotenv>=1.0.1",
101 | # cli requirements
102 | "yaspin>=3.1.0",
103 | # gemini requirements
104 | "google-genai>=1.41.0",
105 | # qwen requirements
106 | "qwen-vl-utils",
107 | "qwen-agent",
108 | "Pillow>=10.0.0",
109 | ]
110 |
111 | [tool.uv]
112 | constraint-dependencies = ["fastrtc>0.43.0", "mlx-audio>0.2.3"]
113 |
114 | [tool.pdm]
115 | distribution = true
116 |
117 | [tool.pdm.build]
118 | includes = ["agent/"]
```
--------------------------------------------------------------------------------
/libs/python/computer/tests/test_computer.py:
--------------------------------------------------------------------------------
```python
1 | """Unit tests for Computer class.
2 |
3 | This file tests ONLY the Computer class initialization and context manager.
4 | Following SRP: This file tests ONE class (Computer).
5 | All external dependencies (providers, interfaces) are mocked.
6 | """
7 |
8 | from unittest.mock import AsyncMock, MagicMock, Mock, patch
9 |
10 | import pytest
11 |
12 |
13 | class TestComputerImport:
14 | """Test Computer module imports (SRP: Only tests imports)."""
15 |
16 | def test_computer_class_exists(self):
17 | """Test that Computer class can be imported."""
18 | from computer import Computer
19 |
20 | assert Computer is not None
21 |
22 | def test_vm_provider_type_exists(self):
23 | """Test that VMProviderType enum can be imported."""
24 | from computer import VMProviderType
25 |
26 | assert VMProviderType is not None
27 |
28 |
29 | class TestComputerInitialization:
30 | """Test Computer initialization (SRP: Only tests initialization)."""
31 |
32 | def test_computer_class_can_be_imported(self, disable_telemetry):
33 | """Test that Computer class can be imported without errors."""
34 | from computer import Computer
35 |
36 | assert Computer is not None
37 |
38 | def test_computer_has_required_methods(self, disable_telemetry):
39 | """Test that Computer class has required methods."""
40 | from computer import Computer
41 |
42 | assert hasattr(Computer, "__aenter__")
43 | assert hasattr(Computer, "__aexit__")
44 |
45 |
46 | class TestComputerContextManager:
47 | """Test Computer context manager protocol (SRP: Only tests context manager)."""
48 |
49 | def test_computer_is_async_context_manager(self, disable_telemetry):
50 | """Test that Computer has async context manager methods."""
51 | from computer import Computer
52 |
53 | assert hasattr(Computer, "__aenter__")
54 | assert hasattr(Computer, "__aexit__")
55 | assert callable(Computer.__aenter__)
56 | assert callable(Computer.__aexit__)
57 |
58 |
59 | class TestComputerInterface:
60 | """Test Computer.interface property (SRP: Only tests interface access)."""
61 |
62 | def test_computer_class_structure(self, disable_telemetry):
63 | """Test that Computer class has expected structure."""
64 | from computer import Computer
65 |
66 | # Verify Computer is a class
67 | assert isinstance(Computer, type)
68 |
```
--------------------------------------------------------------------------------
/libs/kasm/src/ubuntu/install/firefox/custom_startup.sh:
--------------------------------------------------------------------------------
```bash
1 | #!/usr/bin/env bash
2 | set -ex
3 | START_COMMAND="firefox"
4 | PGREP="firefox"
5 | export MAXIMIZE="true"
6 | export MAXIMIZE_NAME="Mozilla Firefox"
7 | MAXIMIZE_SCRIPT=$STARTUPDIR/maximize_window.sh
8 | DEFAULT_ARGS=""
9 | ARGS=${APP_ARGS:-$DEFAULT_ARGS}
10 |
11 | options=$(getopt -o gau: -l go,assign,url: -n "$0" -- "$@") || exit
12 | eval set -- "$options"
13 |
14 | while [[ $1 != -- ]]; do
15 | case $1 in
16 | -g|--go) GO='true'; shift 1;;
17 | -a|--assign) ASSIGN='true'; shift 1;;
18 | -u|--url) OPT_URL=$2; shift 2;;
19 | *) echo "bad option: $1" >&2; exit 1;;
20 | esac
21 | done
22 | shift
23 |
24 | # Process non-option arguments.
25 | for arg; do
26 | echo "arg! $arg"
27 | done
28 |
29 | FORCE=$2
30 |
31 | # run with vgl if GPU is available
32 | if [ -f /opt/VirtualGL/bin/vglrun ] && [ ! -z "${KASM_EGL_CARD}" ] && [ ! -z "${KASM_RENDERD}" ] && [ -O "${KASM_RENDERD}" ] && [ -O "${KASM_EGL_CARD}" ] ; then
33 | START_COMMAND="/opt/VirtualGL/bin/vglrun -d ${KASM_EGL_CARD} $START_COMMAND"
34 | fi
35 |
36 | kasm_exec() {
37 | if [ -n "$OPT_URL" ] ; then
38 | URL=$OPT_URL
39 | elif [ -n "$1" ] ; then
40 | URL=$1
41 | fi
42 |
43 | # Since we are execing into a container that already has the browser running from startup,
44 | # when we don't have a URL to open we want to do nothing. Otherwise a second browser instance would open.
45 | if [ -n "$URL" ] ; then
46 | /usr/bin/filter_ready
47 | /usr/bin/desktop_ready
48 | bash ${MAXIMIZE_SCRIPT} &
49 | $START_COMMAND $ARGS $OPT_URL
50 | else
51 | echo "No URL specified for exec command. Doing nothing."
52 | fi
53 | }
54 |
55 | kasm_startup() {
56 | if [ -n "$KASM_URL" ] ; then
57 | URL=$KASM_URL
58 | elif [ -z "$URL" ] ; then
59 | URL=$LAUNCH_URL
60 | fi
61 |
62 | if [ -z "$DISABLE_CUSTOM_STARTUP" ] || [ -n "$FORCE" ] ; then
63 |
64 | echo "Entering process startup loop"
65 | set +x
66 | while true
67 | do
68 | if ! pgrep -x $PGREP > /dev/null
69 | then
70 | /usr/bin/filter_ready
71 | /usr/bin/desktop_ready
72 | set +e
73 | bash ${MAXIMIZE_SCRIPT} &
74 | $START_COMMAND $ARGS $URL
75 | set -e
76 | fi
77 | sleep 1
78 | done
79 | set -x
80 |
81 | fi
82 |
83 | }
84 |
85 | if [ -n "$GO" ] || [ -n "$ASSIGN" ] ; then
86 | kasm_exec
87 | else
88 | kasm_startup
89 | fi
90 |
```
--------------------------------------------------------------------------------
/libs/typescript/cua-cli/src/auth.ts:
--------------------------------------------------------------------------------
```typescript
1 | import { AUTH_PAGE, CALLBACK_HOST } from './config';
2 | import { setApiKey, getApiKey } from './storage';
3 | import { openInBrowser } from './util';
4 |
5 | const c = {
6 | reset: '\x1b[0m',
7 | bold: '\x1b[1m',
8 | dim: '\x1b[2m',
9 | underline: '\x1b[4m',
10 | cyan: '\x1b[36m',
11 | green: '\x1b[32m',
12 | yellow: '\x1b[33m',
13 | };
14 |
15 | export async function loginViaBrowser(): Promise<string> {
16 | let resolveToken!: (v: string) => void;
17 | const tokenPromise = new Promise<string>((resolve) => {
18 | resolveToken = resolve;
19 | });
20 |
21 | // dynamic port (0) -> OS chooses available port
22 | const server = Bun.serve({
23 | hostname: CALLBACK_HOST,
24 | port: 0,
25 | fetch(req) {
26 | const u = new URL(req.url);
27 | if (u.pathname !== '/callback')
28 | return new Response('Not found', { status: 404 });
29 | const token = u.searchParams.get('token');
30 | if (!token) return new Response('Missing token', { status: 400 });
31 | resolveToken(token);
32 | queueMicrotask(() => server.stop());
33 | return new Response('CLI authorized. You can close this window.', {
34 | status: 200,
35 | headers: { 'content-type': 'text/plain' },
36 | });
37 | },
38 | });
39 |
40 | const callbackURL = `http://${CALLBACK_HOST}:${server.port}/callback`;
41 | const url = `${AUTH_PAGE}?callback_url=${encodeURIComponent(callbackURL)}`;
42 | console.log(
43 | `${c.cyan}${c.bold}Opening your default browser to authorize the CLI...${c.reset}`
44 | );
45 | console.log(
46 | `${c.dim}If the browser does not open automatically, copy/paste this URL:${c.reset}`
47 | );
48 | console.log(`${c.yellow}${c.underline}${url}${c.reset}`);
49 | await openInBrowser(url);
50 |
51 | let timeoutId: ReturnType<typeof setTimeout> | undefined;
52 | const timeout = new Promise<string>((_, reject) => {
53 | timeoutId = setTimeout(
54 | () => reject(new Error('Timed out waiting for authorization')),
55 | 2 * 60 * 1000
56 | );
57 | });
58 | try {
59 | const result = await Promise.race([tokenPromise, timeout]);
60 | if (timeoutId) clearTimeout(timeoutId);
61 | return result;
62 | } finally {
63 | try {
64 | server.stop();
65 | } catch {}
66 | }
67 | }
68 |
69 | export async function ensureApiKeyInteractive(): Promise<string> {
70 | const existing = getApiKey();
71 | if (existing) return existing;
72 | const token = await loginViaBrowser();
73 | setApiKey(token);
74 | return token;
75 | }
76 |
```
--------------------------------------------------------------------------------
/docs/content/docs/agent-sdk/callbacks/logging.mdx:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | title: Logging
3 | description: Agent logging and custom logger implementation
4 | ---
5 |
6 | # Logging Callback
7 |
8 | Built-in logging callback and custom logger creation for agent monitoring.
9 |
10 | ## Callbacks Example
11 |
12 | ```python
13 | from agent.callbacks import LoggingCallback
14 | import logging
15 |
16 | agent = ComputerAgent(
17 | model="anthropic/claude-sonnet-4-5-20250929",
18 | tools=[computer],
19 | callbacks=[
20 | LoggingCallback(
21 | logger=logging.getLogger("cua"),
22 | level=logging.INFO
23 | )
24 | ]
25 | )
26 | ```
27 |
28 | ## Shorthand
29 |
30 | ```python
31 | agent = ComputerAgent(
32 | model="anthropic/claude-sonnet-4-5-20250929",
33 | tools=[computer],
34 | verbosity=logging.INFO # Auto-adds LoggingCallback
35 | )
36 | ```
37 |
38 | ## Custom Logger
39 |
40 | Create custom loggers by extending AsyncCallbackHandler:
41 |
42 | ```python
43 | from agent.callbacks.base import AsyncCallbackHandler
44 | import logging
45 |
46 | class CustomLogger(AsyncCallbackHandler):
47 | def __init__(self, logger_name="agent"):
48 | self.logger = logging.getLogger(logger_name)
49 | self.logger.setLevel(logging.INFO)
50 |
51 | # Add console handler
52 | handler = logging.StreamHandler()
53 | formatter = logging.Formatter(
54 | '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
55 | )
56 | handler.setFormatter(formatter)
57 | self.logger.addHandler(handler)
58 |
59 | async def on_run_start(self, kwargs, old_items):
60 | self.logger.info(f"Agent run started with model: {kwargs.get('model')}")
61 |
62 | async def on_computer_call_start(self, item):
63 | action = item.get('action', {})
64 | self.logger.info(f"Computer action: {action.get('type')}")
65 |
66 | async def on_usage(self, usage):
67 | cost = usage.get('response_cost', 0)
68 | self.logger.info(f"API call cost: ${cost:.4f}")
69 |
70 | async def on_run_end(self, kwargs, old_items, new_items):
71 | self.logger.info("Agent run completed")
72 |
73 | # Use custom logger
74 | agent = ComputerAgent(
75 | model="anthropic/claude-sonnet-4-5-20250929",
76 | tools=[computer],
77 | callbacks=[CustomLogger("my_agent")]
78 | )
79 | ```
80 |
81 | ## Available Hooks
82 |
83 | Log any agent event using these callback methods:
84 |
85 | - `on_run_start/end` - Run lifecycle
86 | - `on_computer_call_start/end` - Computer actions
87 | - `on_api_start/end` - LLM API calls
88 | - `on_usage` - Cost tracking
89 | - `on_screenshot` - Screenshot events
90 |
```
--------------------------------------------------------------------------------
/libs/python/mcp-server/QUICK_TEST_COMMANDS.sh:
--------------------------------------------------------------------------------
```bash
1 | #!/bin/bash
2 | # Quick Test Commands for MCP Server Local Desktop Option
3 | # Run these commands to test the implementation
4 |
5 | set -e # Exit on error
6 |
7 | echo "======================================================================"
8 | echo "Testing MCP Server Local Desktop Option"
9 | echo "======================================================================"
10 | echo ""
11 |
12 | # Change to repo root
13 | cd "$(dirname "$0")/.."
14 |
15 | # Test 1: Quick Logic Test (No setup required)
16 | echo "Test 1: Quick Logic Test (No setup required)"
17 | echo "----------------------------------------------------------------------"
18 | python tests/quick_test_local_option.py
19 | echo ""
20 |
21 | # Test 2: Automated Tests (Requires pytest and packages)
22 | echo "Test 2: Automated Tests (Requires pytest and packages installed)"
23 | echo "----------------------------------------------------------------------"
24 | if command -v pytest &> /dev/null; then
25 | echo "Running pytest..."
26 | pytest tests/test_mcp_server_local_option.py -v || echo "Note: Some tests may require full setup"
27 | else
28 | echo "⚠️ pytest not found. Install with: pip install pytest"
29 | fi
30 | echo ""
31 |
32 | # Test 3: Existing MCP server tests
33 | echo "Test 3: Existing MCP Server Tests"
34 | echo "----------------------------------------------------------------------"
35 | if command -v pytest &> /dev/null; then
36 | echo "Running existing session management tests..."
37 | pytest tests/test_mcp_server_session_management.py -v || echo "Note: Some tests may fail if dependencies are missing"
38 | else
39 | echo "⚠️ pytest not found. Install with: pip install pytest"
40 | fi
41 | echo ""
42 |
43 | # Summary
44 | echo "======================================================================"
45 | echo "Test Summary"
46 | echo "======================================================================"
47 | echo "✅ Quick logic test completed"
48 | echo ""
49 | echo "Next steps for comprehensive testing:"
50 | echo "1. Install dependencies:"
51 | echo " pip install -e libs/python/core"
52 | echo " pip install -e libs/python/computer"
53 | echo " pip install -e libs/python/agent"
54 | echo " pip install -e libs/python/mcp-server"
55 | echo " pip install -e libs/python/computer-server"
56 | echo ""
57 | echo "2. For manual end-to-end testing, see:"
58 | echo " tests/MANUAL_TEST_LOCAL_OPTION.md"
59 | echo ""
60 | echo "3. For detailed testing info, see:"
61 | echo " tests/TESTING_SUMMARY.md"
62 | echo ""
63 |
64 |
```
--------------------------------------------------------------------------------
/.github/scripts/get_pyproject_version.py:
--------------------------------------------------------------------------------
```python
1 | #!/usr/bin/env python3
2 | """
3 | Verifies that the version in pyproject.toml matches the expected version.
4 |
5 | Usage:
6 | python get_pyproject_version.py <pyproject_path> <expected_version>
7 |
8 | Exit codes:
9 | 0 - Versions match
10 | 1 - Versions don't match or error occurred
11 | """
12 |
13 | import sys
14 |
15 | try:
16 | import tomllib
17 | except ImportError:
18 | # Fallback for Python < 3.11
19 | import toml as tomllib
20 |
21 |
22 | def main():
23 | if len(sys.argv) != 3:
24 | print(
25 | "Usage: python get_pyproject_version.py <pyproject_path> <expected_version>",
26 | file=sys.stderr,
27 | )
28 | sys.exit(1)
29 |
30 | pyproject_path = sys.argv[1]
31 | expected_version = sys.argv[2]
32 |
33 | # tomllib requires binary mode
34 | try:
35 | with open(pyproject_path, "rb") as f:
36 | data = tomllib.load(f)
37 | except FileNotFoundError:
38 | print(f"❌ ERROR: File not found: {pyproject_path}", file=sys.stderr)
39 | sys.exit(1)
40 | except Exception as e:
41 | # Fallback to toml if using the old library or handle other errors
42 | try:
43 | import toml
44 |
45 | data = toml.load(pyproject_path)
46 | except FileNotFoundError:
47 | print(f"❌ ERROR: File not found: {pyproject_path}", file=sys.stderr)
48 | sys.exit(1)
49 | except Exception as toml_err:
50 | print(f"❌ ERROR: Failed to parse TOML file: {e}", file=sys.stderr)
51 | sys.exit(1)
52 |
53 | actual_version = data.get("project", {}).get("version")
54 |
55 | if not actual_version:
56 | print("❌ ERROR: No version found in pyproject.toml", file=sys.stderr)
57 | sys.exit(1)
58 |
59 | if actual_version != expected_version:
60 | print("❌ Version mismatch detected!", file=sys.stderr)
61 | print(f" pyproject.toml version: {actual_version}", file=sys.stderr)
62 | print(f" Expected version: {expected_version}", file=sys.stderr)
63 | print("", file=sys.stderr)
64 | print(
65 | "The version in pyproject.toml must match the version being published.", file=sys.stderr
66 | )
67 | print(
68 | f"Please update pyproject.toml to version {expected_version} or use the correct tag.",
69 | file=sys.stderr,
70 | )
71 | sys.exit(1)
72 |
73 | print(f"✅ Version consistency check passed: {actual_version}")
74 | sys.exit(0)
75 |
76 |
77 | if __name__ == "__main__":
78 | main()
79 |
```
--------------------------------------------------------------------------------
/docs/content/docs/agent-sdk/chat-history.mdx:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | title: Chat History
3 | description: Managing conversation history and message arrays
4 | ---
5 |
6 | Managing conversation history is essential for multi-turn agent interactions. The agent maintains a messages array that tracks the entire conversation flow.
7 |
8 | ## Managing History
9 |
10 | ### Continuous Conversation
11 |
12 | ```python
13 | history = []
14 |
15 | while True:
16 | user_input = input("> ")
17 | history.append({"role": "user", "content": user_input})
18 |
19 | async for result in agent.run(history, stream=False):
20 | history += result["output"]
21 | ```
22 |
23 | ## Message Array Structure
24 |
25 | The messages array contains different types of messages that represent the conversation state:
26 |
27 | ```python
28 | messages = [
29 | # user input
30 | {
31 | "role": "user",
32 | "content": "go to trycua on gh"
33 | },
34 | # first agent turn adds the model output to the history
35 | {
36 | "summary": [
37 | {
38 | "text": "Searching Firefox for Trycua GitHub",
39 | "type": "summary_text"
40 | }
41 | ],
42 | "type": "reasoning"
43 | },
44 | {
45 | "action": {
46 | "text": "Trycua GitHub",
47 | "type": "type"
48 | },
49 | "call_id": "call_QI6OsYkXxl6Ww1KvyJc4LKKq",
50 | "status": "completed",
51 | "type": "computer_call"
52 | },
53 | # second agent turn adds the computer output to the history
54 | {
55 | "type": "computer_call_output",
56 | "call_id": "call_QI6OsYkXxl6Ww1KvyJc4LKKq",
57 | "output": {
58 | "type": "input_image",
59 | "image_url": "data:image/png;base64,..."
60 | }
61 | },
62 | # final agent turn adds the agent output text to the history
63 | {
64 | "type": "message",
65 | "role": "assistant",
66 | "content": [
67 | {
68 | "text": "Success! The Trycua GitHub page has been opened.",
69 | "type": "output_text"
70 | }
71 | ]
72 | }
73 | ]
74 | ```
75 |
76 | ## Message Types
77 |
78 | See the complete schema in [Message Format](./message-format).
79 |
80 | ### Memory Management
81 |
82 | For long conversations, consider using the `only_n_most_recent_images` parameter to manage memory:
83 |
84 | ```python
85 | agent = ComputerAgent(
86 | model="cua/anthropic/claude-sonnet-4.5",
87 | tools=[computer],
88 | only_n_most_recent_images=3
89 | )
90 | ```
91 |
92 | This automatically removes old images from the conversation history to prevent context window overflow.
93 |
```
--------------------------------------------------------------------------------
/TESTING.md:
--------------------------------------------------------------------------------
```markdown
1 | # Testing Guide for CUA
2 |
3 | Quick guide to running tests and understanding the test architecture.
4 |
5 | ## 🚀 Quick Start
6 |
7 | ```bash
8 | # Install dependencies
9 | pip install pytest pytest-asyncio pytest-mock pytest-cov
10 |
11 | # Install package
12 | cd libs/python/core
13 | pip install -e .
14 |
15 | # Run tests
16 | export CUA_TELEMETRY_DISABLED=1 # or $env:CUA_TELEMETRY_DISABLED="1" on Windows
17 | pytest tests/ -v
18 | ```
19 |
20 | ## 🧪 Running Tests
21 |
22 | ```bash
23 | # All packages
24 | pytest libs/python/*/tests/ -v
25 |
26 | # Specific package
27 | cd libs/python/core && pytest tests/ -v
28 |
29 | # With coverage
30 | pytest tests/ --cov --cov-report=html
31 |
32 | # Specific test
33 | pytest tests/test_telemetry.py::TestTelemetryEnabled::test_telemetry_enabled_by_default -v
34 | ```
35 |
36 | ## 🏗️ Test Architecture
37 |
38 | **Principles**: SRP (Single Responsibility) + Vertical Slices + Testability
39 |
40 | ```
41 | libs/python/
42 | ├── core/tests/ # Tests ONLY core
43 | ├── agent/tests/ # Tests ONLY agent
44 | └── computer/tests/ # Tests ONLY computer
45 | ```
46 |
47 | Each test file = ONE feature. Each test class = ONE concern.
48 |
49 | ## ➕ Adding New Tests
50 |
51 | 1. Create `test_*.py` in the appropriate package's `tests/` directory
52 | 2. Follow the pattern:
53 |
54 | ```python
55 | """Unit tests for my_feature."""
56 | import pytest
57 | from unittest.mock import patch
58 |
59 | class TestMyFeature:
60 | """Test MyFeature class."""
61 |
62 | def test_initialization(self):
63 | """Test that feature initializes."""
64 | from my_package import MyFeature
65 | feature = MyFeature()
66 | assert feature is not None
67 | ```
68 |
69 | 3. Mock external dependencies:
70 |
71 | ```python
72 | @pytest.fixture
73 | def mock_api():
74 | with patch("my_package.api_client") as mock:
75 | yield mock
76 | ```
77 |
78 | ## 🔄 CI/CD
79 |
80 | Tests run automatically on every PR via GitHub Actions (`.github/workflows/python-tests.yml`):
81 |
82 | - Matrix strategy: each package tested separately
83 | - Python 3.12
84 | - ~2 minute runtime
85 |
86 | ## 🐛 Troubleshooting
87 |
88 | **ModuleNotFoundError**: Run `pip install -e .` in package directory
89 |
90 | **Tests fail in CI but pass locally**: Set `CUA_TELEMETRY_DISABLED=1`
91 |
92 | **Async tests error**: Install `pytest-asyncio` and use `@pytest.mark.asyncio`
93 |
94 | **Mock not working**: Patch at usage location, not definition:
95 |
96 | ```python
97 | # ✅ Right
98 | @patch("my_package.module.external_function")
99 |
100 | # ❌ Wrong
101 | @patch("external_library.function")
102 | ```
103 |
104 | ---
105 |
106 | **Questions?** Check existing tests for examples or open an issue.
107 |
```
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
```toml
1 | [project]
2 | authors = [{ name = "TryCua", email = "[email protected]" }]
3 | dependencies = [
4 | "openai<1.100.0",
5 | "anthropic>=0.67.0",
6 | ]
7 | description = "CUA (Computer Use Agent) mono-repo"
8 | license = { text = "MIT" }
9 | name = "cua-workspace"
10 | readme = "README.md"
11 | requires-python = "<3.14,>=3.12"
12 | version = "0.1.0"
13 |
14 | [project.urls]
15 | repository = "https://github.com/trycua/cua"
16 |
17 | [dependency-groups]
18 | dev = [
19 | "cua-core",
20 | "cua-agent",
21 | "cua-computer",
22 | "cua-computer-server",
23 | "cua-som",
24 | "cua-mcp-server",
25 | "black>=23.0.0",
26 | "ipykernel>=6.29.5",
27 | "jedi>=0.19.2",
28 | "jupyter>=1.0.0",
29 | "mypy>=1.10.0",
30 | "ruff>=0.9.2",
31 | "types-requests>=2.31.0",
32 | "hud-python[agent]==0.4.52",
33 | "pre-commit>=4.3.0",
34 | "isort>=7.0.0",
35 | ]
36 | docs = [
37 | "mkdocs-material>=9.2.0",
38 | "mkdocs>=1.5.0",
39 | ]
40 | test = [
41 | "aioresponses>=0.7.4",
42 | "pytest-asyncio>=0.21.1",
43 | "pytest-cov>=4.1.0",
44 | "pytest-mock>=3.10.0",
45 | "pytest-xdist>=3.6.1",
46 | "pytest>=8.0.0",
47 | ]
48 | examples = []
49 |
50 | [tool.uv]
51 | package = false
52 |
53 | [tool.uv.workspace]
54 | members = [
55 | "libs/python/agent",
56 | "libs/python/core",
57 | "libs/python/computer",
58 | "libs/python/computer-server",
59 | "libs/python/som",
60 | "libs/python/mcp-server",
61 | "libs/python/bench-ui",
62 | ]
63 |
64 | [tool.uv.sources]
65 | cua-agent = { workspace = true }
66 | cua-core = { workspace = true }
67 | cua-computer = { workspace = true }
68 | cua-computer-server = { workspace = true }
69 | cua-som = { workspace = true }
70 | cua-mcp-server = { workspace = true }
71 | cua-bench-ui = { workspace = true }
72 |
73 | [tool.black]
74 | line-length = 100
75 | target-version = ["py312"]
76 |
77 | [tool.ruff]
78 | fix = true
79 | line-length = 100
80 | target-version = "py312"
81 |
82 | [tool.ruff.lint]
83 | select = ["E", "F", "B", "I"]
84 | ignore = [
85 | "E501", "E402", "I001", "I002", "B007", "B023", "B024", "B027", "B028",
86 | "B904", "B905", "E711", "E712", "E722", "E731", "F401", "F403", "F405",
87 | "F811", "F821", "F841"
88 | ]
89 |
90 | [tool.ruff.format]
91 | docstring-code-format = true
92 |
93 | [tool.mypy]
94 | check_untyped_defs = true
95 | disallow_untyped_defs = true
96 | ignore_missing_imports = true
97 | python_version = "3.12"
98 | show_error_codes = true
99 | strict = true
100 | warn_return_any = true
101 | warn_unused_ignores = false
102 |
103 | [tool.isort]
104 | profile = "black"
105 | py_version = 312
106 |
107 | [tool.pytest.ini_options]
108 | asyncio_mode = "auto"
109 | python_files = "test_*.py"
110 | testpaths = ["libs/*/tests"]
111 |
```
--------------------------------------------------------------------------------
/examples/cloud_api_examples.py:
--------------------------------------------------------------------------------
```python
1 | import asyncio
2 | import os
3 |
4 | from utils import load_dotenv_files
5 |
6 | load_dotenv_files()
7 |
8 | from computer.providers.cloud.provider import CloudProvider
9 |
10 |
11 | async def main() -> None:
12 | # CloudProvider will automatically read CUA_API_KEY from environment if not provided
13 | # You can still pass api_key explicitly if needed: CloudProvider(api_key="your_key")
14 | api_base = os.getenv("CUA_API_BASE")
15 | if api_base:
16 | print(f"Using API base: {api_base}")
17 |
18 | provider = CloudProvider(verbose=True)
19 | async with provider:
20 |
21 | # List all VMs
22 | vms = await provider.list_vms()
23 | print(f"Found {len(vms)} VM(s)")
24 | for vm in vms:
25 | print(
26 | f"name: {vm['name']}\n",
27 | f"status: {vm['status']}\n", # pending, running, stopped, terminated, failed
28 | f"api_url: {vm.get('api_url')}\n",
29 | f"vnc_url: {vm.get('vnc_url')}\n",
30 | )
31 |
32 | # # --- Additional operations (commented out) ---
33 | # # To stop a VM by name:
34 | # name = "m-linux-96lcxd2c2k"
35 | # resp = await provider.stop_vm(name)
36 | # print(
37 | # "stop_vm response:\n",
38 | # f"name: {resp['name']}\n",
39 | # f"status: {resp['status']}\n", # stopping
40 | # )
41 |
42 | # # To start a VM by name:
43 | # name = "m-linux-96lcxd2c2k"
44 | # resp = await provider.run_vm(name)
45 | # print(
46 | # "run_vm response:\n",
47 | # f"name: {resp['name']}\n",
48 | # f"status: {resp['status']}\n", # starting
49 | # )
50 |
51 | # # To restart a VM by name:
52 | # name = "m-linux-96lcxd2c2k"
53 | # resp = await provider.restart_vm(name)
54 | # print(
55 | # "restart_vm response:\n",
56 | # f"name: {resp['name']}\n",
57 | # f"status: {resp['status']}\n", # restarting
58 | # )
59 |
60 | # # To probe a VM's status via its public hostname (if you know the name):
61 | # name = "m-linux-96lcxd2c2k"
62 | # info = await provider.get_vm(name)
63 | # print("get_vm info:\n",
64 | # f"name: {info['name']}\n",
65 | # f"status: {info['status']}\n", # running
66 | # f"api_url: {info.get('api_url')}\n",
67 | # f"os_type: {info.get('os_type')}\n",
68 | # )
69 |
70 |
71 | if __name__ == "__main__":
72 | asyncio.run(main())
73 |
```
--------------------------------------------------------------------------------
/libs/python/bench-ui/examples/simple_example.py:
--------------------------------------------------------------------------------
```python
1 | from __future__ import annotations
2 |
3 | import os
4 | import time
5 | from pathlib import Path
6 |
7 | from bench_ui import execute_javascript, get_element_rect, launch_window
8 |
9 | HTML = """
10 | <!doctype html>
11 | <html>
12 | <head>
13 | <meta charset="utf-8" />
14 | <title>Bench UI Example</title>
15 | <style>
16 | body { font-family: system-ui, sans-serif; margin: 24px; }
17 | #target { width: 220px; height: 120px; background: #4f46e5; color: white; display: flex; align-items: center; justify-content: center; border-radius: 8px; }
18 | </style>
19 | </head>
20 | <body>
21 | <h1>Bench UI Example</h1>
22 | <div id="target">Hello from pywebview</div>
23 |
24 |
25 | <h1>Click the button</h1>
26 | <button id="submit" class="btn" data-instruction="the button">Submit</button>
27 | <script>
28 | window.__submitted = false;
29 | document.getElementById('submit').addEventListener('click', function() {
30 | window.__submitted = true;
31 | this.textContent = 'Submitted!';
32 | this.disabled = true;
33 | });
34 | </script>
35 | </body>
36 | </html>
37 | """
38 |
39 |
40 | def main():
41 | os.environ["CUA_BENCH_UI_DEBUG"] = "1"
42 |
43 | # Launch a window with inline HTML content
44 | pid = launch_window(
45 | html=HTML,
46 | title="Bench UI Example",
47 | width=800,
48 | height=600,
49 | )
50 | print(f"Launched window with PID: {pid}")
51 |
52 | # Give the window a brief moment to render
53 | time.sleep(1.0)
54 |
55 | # Query the client rect of an element via CSS selector in SCREEN space
56 | rect = get_element_rect(pid, "#target", space="screen")
57 | print("Element rect (screen space):", rect)
58 |
59 | # Take a screenshot and overlay the bbox
60 | try:
61 | from PIL import ImageDraw, ImageGrab
62 |
63 | img = ImageGrab.grab() # full screen
64 | draw = ImageDraw.Draw(img)
65 | x, y, w, h = rect["x"], rect["y"], rect["width"], rect["height"]
66 | box = (x, y, x + w, y + h)
67 | draw.rectangle(box, outline=(255, 0, 0), width=3)
68 | out_path = Path(__file__).parent / "output_overlay.png"
69 | img.save(out_path)
70 | print(f"Saved overlay screenshot to: {out_path}")
71 | except Exception as e:
72 | print(f"Failed to capture/annotate screenshot: {e}")
73 |
74 | # Execute arbitrary JavaScript
75 | text = execute_javascript(pid, "window.__submitted")
76 | print("text:", text)
77 |
78 |
79 | if __name__ == "__main__":
80 | main()
81 |
```
--------------------------------------------------------------------------------
/.github/workflows/pypi-publish-computer-server.yml:
--------------------------------------------------------------------------------
```yaml
1 | name: Publish Computer Server Package
2 |
3 | on:
4 | push:
5 | tags:
6 | - "computer-server-v*"
7 | workflow_dispatch:
8 | inputs:
9 | version:
10 | description: "Version to publish (without v prefix)"
11 | required: true
12 | default: "0.1.0"
13 | workflow_call:
14 | inputs:
15 | version:
16 | description: "Version to publish"
17 | required: true
18 | type: string
19 | outputs:
20 | version:
21 | description: "The version that was published"
22 | value: ${{ jobs.prepare.outputs.version }}
23 |
24 | # Adding permissions at workflow level
25 | permissions:
26 | contents: write
27 |
28 | jobs:
29 | prepare:
30 | runs-on: macos-latest
31 | outputs:
32 | version: ${{ steps.get-version.outputs.version }}
33 | steps:
34 | - uses: actions/checkout@v4
35 |
36 | - name: Determine version
37 | id: get-version
38 | run: |
39 | if [ "${{ github.event_name }}" == "push" ]; then
40 | # Extract version from tag (for package-specific tags)
41 | if [[ "${{ github.ref }}" =~ ^refs/tags/computer-server-v([0-9]+\.[0-9]+\.[0-9]+) ]]; then
42 | VERSION=${BASH_REMATCH[1]}
43 | else
44 | echo "Invalid tag format for computer-server"
45 | exit 1
46 | fi
47 | elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
48 | # Use version from workflow dispatch
49 | VERSION=${{ github.event.inputs.version }}
50 | else
51 | # Use version from workflow_call
52 | VERSION=${{ inputs.version }}
53 | fi
54 | echo "VERSION=$VERSION"
55 | echo "version=$VERSION" >> $GITHUB_OUTPUT
56 |
57 | - name: Set up Python
58 | uses: actions/setup-python@v4
59 | with:
60 | python-version: "3.10"
61 |
62 | publish:
63 | needs: prepare
64 | uses: ./.github/workflows/pypi-reusable-publish.yml
65 | with:
66 | package_name: "computer-server"
67 | package_dir: "libs/python/computer-server"
68 | version: ${{ needs.prepare.outputs.version }}
69 | is_lume_package: false
70 | base_package_name: "cua-computer-server"
71 | secrets:
72 | PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
73 |
74 | set-env-variables:
75 | needs: [prepare, publish]
76 | runs-on: macos-latest
77 | steps:
78 | - name: Set environment variables for use in other jobs
79 | run: |
80 | echo "COMPUTER_VERSION=${{ needs.prepare.outputs.version }}" >> $GITHUB_ENV
81 |
```
--------------------------------------------------------------------------------
/libs/python/agent/agent/computers/base.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Base computer interface protocol for agent interactions.
3 | """
4 |
5 | from typing import (
6 | Any,
7 | Dict,
8 | List,
9 | Literal,
10 | Optional,
11 | Protocol,
12 | Union,
13 | runtime_checkable,
14 | )
15 |
16 |
17 | @runtime_checkable
18 | class AsyncComputerHandler(Protocol):
19 | """Protocol defining the interface for computer interactions."""
20 |
21 | # ==== Computer-Use-Preview Action Space ====
22 |
23 | async def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]:
24 | """Get the current environment type."""
25 | ...
26 |
27 | async def get_dimensions(self) -> tuple[int, int]:
28 | """Get screen dimensions as (width, height)."""
29 | ...
30 |
31 | async def screenshot(self, text: Optional[str] = None) -> str:
32 | """Take a screenshot and return as base64 string.
33 |
34 | Args:
35 | text: Optional descriptive text (for compatibility with GPT-4o models, ignored)
36 | """
37 | ...
38 |
39 | async def click(self, x: int, y: int, button: str = "left") -> None:
40 | """Click at coordinates with specified button."""
41 | ...
42 |
43 | async def double_click(self, x: int, y: int) -> None:
44 | """Double click at coordinates."""
45 | ...
46 |
47 | async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
48 | """Scroll at coordinates with specified scroll amounts."""
49 | ...
50 |
51 | async def type(self, text: str) -> None:
52 | """Type text."""
53 | ...
54 |
55 | async def wait(self, ms: int = 1000) -> None:
56 | """Wait for specified milliseconds."""
57 | ...
58 |
59 | async def move(self, x: int, y: int) -> None:
60 | """Move cursor to coordinates."""
61 | ...
62 |
63 | async def keypress(self, keys: Union[List[str], str]) -> None:
64 | """Press key combination."""
65 | ...
66 |
67 | async def drag(self, path: List[Dict[str, int]]) -> None:
68 | """Drag along specified path."""
69 | ...
70 |
71 | async def get_current_url(self) -> str:
72 | """Get current URL (for browser environments)."""
73 | ...
74 |
75 | # ==== Anthropic Action Space ====
76 |
77 | async def left_mouse_down(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
78 | """Left mouse down at coordinates."""
79 | ...
80 |
81 | async def left_mouse_up(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
82 | """Left mouse up at coordinates."""
83 | ...
84 |
```
--------------------------------------------------------------------------------
/libs/lume/src/VNC/PassphraseGenerator.swift:
--------------------------------------------------------------------------------
```swift
1 | import Foundation
2 | import CryptoKit
3 |
4 | final class PassphraseGenerator {
5 | private let words: [String]
6 |
7 | init(words: [String] = PassphraseGenerator.defaultWords) {
8 | self.words = words
9 | }
10 |
11 | func prefix(_ count: Int) -> [String] {
12 | guard count > 0 else { return [] }
13 |
14 | // Use secure random number generation
15 | var result: [String] = []
16 | for _ in 0..<count {
17 | let randomBytes = (0..<4).map { _ in UInt8.random(in: 0...255) }
18 | let randomNumber = Data(randomBytes).withUnsafeBytes { bytes in
19 | bytes.load(as: UInt32.self)
20 | }
21 | let index = Int(randomNumber % UInt32(words.count))
22 | result.append(words[index])
23 | }
24 | return result
25 | }
26 |
27 | // A much larger set of common, easy-to-type words
28 | private static let defaultWords = [
29 | "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel",
30 | "india", "juliet", "kilo", "lima", "mike", "november", "oscar", "papa",
31 | "quebec", "romeo", "sierra", "tango", "uniform", "victor", "whiskey", "xray",
32 | "yankee", "zulu", "zero", "one", "two", "three", "four", "five",
33 | "six", "seven", "eight", "nine", "apple", "banana", "cherry", "date",
34 | "elder", "fig", "grape", "honey", "iris", "jade", "kiwi", "lemon",
35 | "mango", "nectarine", "orange", "peach", "quince", "raspberry", "strawberry", "tangerine",
36 | "red", "blue", "green", "yellow", "purple", "orange", "pink", "brown",
37 | "black", "white", "gray", "silver", "gold", "copper", "bronze", "steel",
38 | "north", "south", "east", "west", "spring", "summer", "autumn", "winter",
39 | "river", "ocean", "mountain", "valley", "forest", "desert", "island", "beach",
40 | "sun", "moon", "star", "cloud", "rain", "snow", "wind", "storm",
41 | "happy", "brave", "calm", "swift", "wise", "kind", "bold", "free",
42 | "safe", "strong", "bright", "clear", "light", "soft", "warm", "cool",
43 | "eagle", "falcon", "hawk", "owl", "robin", "sparrow", "swan", "dove",
44 | "tiger", "lion", "bear", "wolf", "deer", "horse", "dolphin", "whale",
45 | "maple", "oak", "pine", "birch", "cedar", "fir", "palm", "willow",
46 | "rose", "lily", "daisy", "tulip", "lotus", "orchid", "violet", "jasmine"
47 | ]
48 | }
```
--------------------------------------------------------------------------------
/docs/content/docs/agent-sdk/benchmarks/introduction.mdx:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | title: Introduction
3 | description: Overview of benchmarking in the Cua agent framework
4 | ---
5 |
6 | The Cua agent framework uses benchmarks to test the performance of supported models and providers at various agentic tasks.
7 |
8 | ## Benchmark Types
9 |
10 | Computer-Agent benchmarks evaluate two key capabilities:
11 |
12 | - **Plan Generation**: Breaking down complex tasks into a sequence of actions
13 | - **Coordinate Generation**: Predicting precise click locations on GUI elements
14 |
15 | ## Using State-of-the-Art Models
16 |
17 | Let's see how to use the SOTA vision-language models in the Cua agent framework.
18 |
19 | ### Plan Generation + Coordinate Generation
20 |
21 | **[OS-World](https://os-world.github.io/)** - Benchmark for complete computer-use agents
22 |
23 | This leaderboard tests models that can understand instructions and automatically perform the full sequence of actions needed to complete tasks.
24 |
25 | ```python
26 | # UI-TARS-1.5 is a SOTA unified plan generation + coordinate generation VLM
27 | # This makes it suitable for agentic loops for computer-use
28 | agent = ComputerAgent("huggingface-local/ByteDance-Seed/UI-TARS-1.5-7B", tools=[computer])
29 | agent.run("Open Firefox and go to github.com")
30 | # Success! 🎉
31 | ```
32 |
33 | ### Coordinate Generation Only
34 |
35 | **[GUI Agent Grounding Leaderboard](https://gui-agent.github.io/grounding-leaderboard/)** - Benchmark for click prediction accuracy
36 |
37 | This leaderboard tests models that specialize in finding exactly where to click on screen elements, but needs to be told what specific action to take.
38 |
39 | ```python
40 | # GTA1-7B is a SOTA coordinate generation VLM
41 | # It can only generate coordinates, not plan:
42 | agent = ComputerAgent("huggingface-local/HelloKKMe/GTA1-7B", tools=[computer])
43 | agent.predict_click("find the button to open the settings") # (27, 450)
44 | # This will raise an error:
45 | # agent.run("Open Firefox and go to github.com")
46 | ```
47 |
48 | ### Composed Agent
49 |
50 | The Cua agent framework also supports composed agents, which combine a planning model with a clicking model for the best of both worlds. Any liteLLM model can be used as the plan generation model.
51 |
52 | ```python
53 | # It can be paired with any LLM to form a composed agent:
54 | # "gemini/gemini-1.5-pro" will be used as the plan generation LLM
55 | agent = ComputerAgent("huggingface-local/HelloKKMe/GTA1-7B+gemini/gemini-1.5-pro", tools=[computer])
56 | agent.run("Open Firefox and go to github.com")
57 | # Success! 🎉
58 | ```
59 |
```
--------------------------------------------------------------------------------
/libs/python/som/tests/test_omniparser.py:
--------------------------------------------------------------------------------
```python
1 | """Unit tests for som package (Set-of-Mark).
2 |
3 | This file tests ONLY basic som functionality.
4 | Following SRP: This file tests som module imports and basic operations.
5 | All external dependencies (ML models, OCR) are mocked.
6 | """
7 |
8 | import pytest
9 |
10 |
11 | class TestSomImports:
12 | """Test som module imports (SRP: Only tests imports)."""
13 |
14 | def test_som_module_exists(self):
15 | """Test that som module can be imported."""
16 | try:
17 | import som
18 |
19 | assert som is not None
20 | except ImportError:
21 | pytest.skip("som module not installed")
22 |
23 | def test_omniparser_import(self):
24 | """Test that OmniParser can be imported."""
25 | try:
26 | from som import OmniParser
27 |
28 | assert OmniParser is not None
29 | except ImportError:
30 | pytest.skip("som module not available")
31 | except Exception as e:
32 | pytest.skip(f"som initialization requires ML models: {e}")
33 |
34 | def test_models_import(self):
35 | """Test that model classes can be imported."""
36 | try:
37 | from som import BoundingBox, ParseResult, UIElement
38 |
39 | assert BoundingBox is not None
40 | assert UIElement is not None
41 | assert ParseResult is not None
42 | except ImportError:
43 | pytest.skip("som models not available")
44 | except Exception as e:
45 | pytest.skip(f"som models require dependencies: {e}")
46 |
47 |
48 | class TestSomModels:
49 | """Test som data models (SRP: Only tests model structure)."""
50 |
51 | def test_bounding_box_structure(self):
52 | """Test BoundingBox class structure."""
53 | try:
54 | from som import BoundingBox
55 |
56 | # Check the class exists and has expected structure
57 | assert hasattr(BoundingBox, "__init__")
58 | except ImportError:
59 | pytest.skip("som models not available")
60 | except Exception as e:
61 | pytest.skip(f"som models require dependencies: {e}")
62 |
63 | def test_ui_element_structure(self):
64 | """Test UIElement class structure."""
65 | try:
66 | from som import UIElement
67 |
68 | # Check the class exists and has expected structure
69 | assert hasattr(UIElement, "__init__")
70 | except ImportError:
71 | pytest.skip("som models not available")
72 | except Exception as e:
73 | pytest.skip(f"som models require dependencies: {e}")
74 |
```
--------------------------------------------------------------------------------
/libs/python/computer/computer/helpers.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Helper functions and decorators for the Computer module.
3 | """
4 |
5 | import asyncio
6 | import logging
7 | from functools import wraps
8 | from typing import Any, Awaitable, Callable, Optional, TypeVar
9 |
10 | try:
11 | # Python 3.12+ has ParamSpec in typing
12 | from typing import ParamSpec
13 | except ImportError: # pragma: no cover
14 | # Fallback for environments without ParamSpec in typing
15 | from typing_extensions import ParamSpec # type: ignore
16 |
17 | P = ParamSpec("P")
18 | R = TypeVar("R")
19 |
20 | # Global reference to the default computer instance
21 | _default_computer = None
22 |
23 | logger = logging.getLogger(__name__)
24 |
25 |
26 | def set_default_computer(computer: Any) -> None:
27 | """
28 | Set the default computer instance to be used by the remote decorator.
29 |
30 | Args:
31 | computer: The computer instance to use as default
32 | """
33 | global _default_computer
34 | _default_computer = computer
35 |
36 |
37 | def sandboxed(
38 | venv_name: str = "default",
39 | computer: str = "default",
40 | max_retries: int = 3,
41 | ) -> Callable[[Callable[P, R]], Callable[P, Awaitable[R]]]:
42 | """
43 | Decorator that wraps a function to be executed remotely via computer.venv_exec
44 |
45 | Args:
46 | venv_name: Name of the virtual environment to execute in
47 | computer: The computer instance to use, or "default" to use the globally set default
48 | max_retries: Maximum number of retries for the remote execution
49 | """
50 |
51 | def decorator(func: Callable[P, R]) -> Callable[P, Awaitable[R]]:
52 | @wraps(func)
53 | async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
54 | # Determine which computer instance to use
55 | comp = computer if computer != "default" else _default_computer
56 |
57 | if comp is None:
58 | raise RuntimeError(
59 | "No computer instance available. Either specify a computer instance or call set_default_computer() first."
60 | )
61 |
62 | for i in range(max_retries):
63 | try:
64 | return await comp.venv_exec(venv_name, func, *args, **kwargs)
65 | except Exception as e:
66 | logger.error(f"Attempt {i+1} failed: {e}")
67 | await asyncio.sleep(1)
68 | if i == max_retries - 1:
69 | raise e
70 |
71 | # Should be unreachable because we either returned or raised
72 | raise RuntimeError("sandboxed wrapper reached unreachable code path")
73 |
74 | return wrapper
75 |
76 | return decorator
77 |
```
--------------------------------------------------------------------------------
/libs/lume/src/Commands/Push.swift:
--------------------------------------------------------------------------------
```swift
1 | import ArgumentParser
2 | import Foundation
3 |
4 | struct Push: AsyncParsableCommand {
5 | static let configuration = CommandConfiguration(
6 | abstract: "Push a macOS VM to GitHub Container Registry"
7 | )
8 |
9 | @Argument(help: "Name of the VM to push")
10 | var name: String
11 |
12 | @Argument(help: "Image tag to push (format: name:tag)")
13 | var image: String
14 |
15 | @Option(parsing: .upToNextOption, help: "Additional tags to push the same image to")
16 | var additionalTags: [String] = []
17 |
18 | @Option(help: "Github Container Registry to push to. Defaults to ghcr.io")
19 | var registry: String = "ghcr.io"
20 |
21 | @Option(help: "Organization to push to. Defaults to trycua")
22 | var organization: String = "trycua"
23 |
24 | @Option(name: .customLong("storage"), help: "VM storage location to use")
25 | var storage: String?
26 |
27 | @Option(help: "Chunk size for large files in MB. Defaults to 512.")
28 | var chunkSizeMb: Int = 512
29 |
30 | @Flag(name: .long, help: "Enable verbose logging")
31 | var verbose: Bool = false
32 |
33 | @Flag(name: .long, help: "Prepare files without uploading to registry")
34 | var dryRun: Bool = false
35 |
36 | @Flag(name: .long, help: "In dry-run mode, also reassemble chunks to verify integrity")
37 | var reassemble: Bool = true
38 |
39 | init() {}
40 |
41 | @MainActor
42 | func run() async throws {
43 | let controller = LumeController()
44 |
45 | // Parse primary image name and tag
46 | let components = image.split(separator: ":")
47 | guard components.count == 2, let primaryTag = components.last else {
48 | throw ValidationError("Invalid primary image format. Expected format: name:tag")
49 | }
50 | let imageName = String(components.first!)
51 |
52 | // Combine primary and additional tags, ensuring uniqueness
53 | var allTags: Swift.Set<String> = []
54 | allTags.insert(String(primaryTag))
55 | allTags.formUnion(additionalTags)
56 |
57 | guard !allTags.isEmpty else {
58 | throw ValidationError("At least one tag must be provided.")
59 | }
60 |
61 | try await controller.pushImage(
62 | name: name,
63 | imageName: imageName, // Pass base image name
64 | tags: Array(allTags), // Pass array of all unique tags
65 | registry: registry,
66 | organization: organization,
67 | storage: storage,
68 | chunkSizeMb: chunkSizeMb,
69 | verbose: verbose,
70 | dryRun: dryRun,
71 | reassemble: reassemble
72 | )
73 | }
74 | }
```
--------------------------------------------------------------------------------
/libs/typescript/cua-cli/src/commands/auth.ts:
--------------------------------------------------------------------------------
```typescript
1 | import { setApiKey, clearApiKey } from '../storage';
2 | import { ensureApiKeyInteractive, loginViaBrowser } from '../auth';
3 | import { writeEnvFile } from '../util';
4 | import type { Argv } from 'yargs';
5 |
6 | // Command handlers
7 | const loginHandler = async (argv: Record<string, unknown>) => {
8 | if (argv['api-key']) {
9 | setApiKey(String(argv['api-key']));
10 | console.log('API key saved');
11 | return;
12 | }
13 | console.log('Opening browser for CLI auth...');
14 | const token = await loginViaBrowser();
15 | setApiKey(token);
16 | console.log('API key saved');
17 | };
18 |
19 | const envHandler = async (_argv: Record<string, unknown>) => {
20 | const token = await ensureApiKeyInteractive();
21 | const out = await writeEnvFile(process.cwd(), token);
22 | console.log(`Wrote ${out}`);
23 | };
24 |
25 | const logoutHandler = async (_argv: Record<string, unknown>) => {
26 | clearApiKey();
27 | console.log('Logged out');
28 | };
29 |
30 | export function registerAuthCommands(y: Argv) {
31 | // Grouped structure: cua auth <command> (register first to appear first in help)
32 | y.command(
33 | 'auth',
34 | 'Authenticate with CUA (login, logout, or export credentials)',
35 | (y) => {
36 | return y
37 | .command(
38 | 'login',
39 | 'Authenticate via browser or API key and save credentials locally',
40 | (y) =>
41 | y.option('api-key', {
42 | type: 'string',
43 | describe: 'API key to store directly',
44 | }),
45 | loginHandler
46 | )
47 | .command(
48 | 'env',
49 | 'Export your API key to a .env file in the current directory',
50 | () => {},
51 | envHandler
52 | )
53 | .command(
54 | 'logout',
55 | 'Clear stored API credentials from this machine',
56 | () => {},
57 | logoutHandler
58 | )
59 | .demandCommand(1, 'You must provide an auth command');
60 | },
61 | () => {}
62 | );
63 |
64 | // Flat structure (backwards compatible, hidden from help)
65 | y.command({
66 | command: 'login',
67 | describe: false as any, // Hide from help
68 | builder: (y: Argv) =>
69 | y.option('api-key', {
70 | type: 'string',
71 | describe: 'API key to store directly',
72 | }),
73 | handler: loginHandler,
74 | } as any)
75 | .command({
76 | command: 'env',
77 | describe: false as any, // Hide from help
78 | builder: (y: Argv) => y,
79 | handler: envHandler,
80 | } as any)
81 | .command({
82 | command: 'logout',
83 | describe: false as any, // Hide from help
84 | builder: (y: Argv) => y,
85 | handler: logoutHandler,
86 | } as any);
87 |
88 | return y;
89 | }
90 |
```
--------------------------------------------------------------------------------
/docs/content/docs/agent-sdk/mcp-server/tools.mdx:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | title: Tools
3 | ---
4 |
5 | ## Available Tools
6 |
7 | The MCP server exposes the following tools to Claude:
8 |
9 | ### Core Task Execution Tools
10 |
11 | 1. **`run_cua_task`** - Run a single Computer-Use Agent task with the given instruction
12 | - `task` (string): The task description for the agent to execute
13 | - `session_id` (string, optional): Session ID for multi-client support. If not provided, a new session will be created
14 | - Returns: Tuple of (combined text output, final screenshot)
15 |
16 | 2. **`run_multi_cua_tasks`** - Run multiple tasks in sequence or concurrently
17 | - `tasks` (list of strings): List of task descriptions to execute
18 | - `session_id` (string, optional): Session ID for multi-client support. If not provided, a new session will be created
19 | - `concurrent` (boolean, optional): If true, run tasks concurrently. If false, run sequentially (default)
20 | - Returns: List of tuples (combined text output, screenshot) for each task
21 |
22 | ### Utility Tools
23 |
24 | 3. **`screenshot_cua`** - Take a screenshot of the current screen
25 | - `session_id` (string, optional): Session ID for multi-client support. If not provided, a new session will be created
26 | - Returns: Screenshot image
27 |
28 | 4. **`get_session_stats`** - Get statistics about active sessions and resource usage
29 | - Returns: Dictionary with session statistics including total sessions, active tasks, and session details
30 |
31 | 5. **`cleanup_session`** - Cleanup a specific session and release its resources
32 | - `session_id` (string): The session ID to cleanup
33 | - Returns: Confirmation message
34 |
35 | ## Session Management
36 |
37 | The MCP server supports multi-client sessions with automatic resource management:
38 |
39 | - **Session Isolation**: Each client can have its own session with isolated computer instances
40 | - **Resource Pooling**: Computer instances are pooled for efficient resource usage
41 | - **Automatic Cleanup**: Idle sessions are automatically cleaned up after 10 minutes
42 | - **Concurrent Tasks**: Multiple tasks can run concurrently within the same session
43 | - **Progress Reporting**: Real-time progress updates during task execution
44 |
45 | ## Usage Examples
46 |
47 | ### Basic Task Execution
48 |
49 | ```
50 | "Open Chrome and navigate to github.com"
51 | "Create a folder called 'Projects' on my desktop"
52 | ```
53 |
54 | ### Multi-Task Execution
55 |
56 | ```
57 | "Run these tasks: 1) Open Finder, 2) Navigate to Documents, 3) Create a new folder called 'Work'"
58 | ```
59 |
60 | ### Session Management
61 |
62 | ```
63 | "Take a screenshot of the current screen"
64 | "Show me the session statistics"
65 | "Cleanup session abc123"
66 | ```
67 |
```
--------------------------------------------------------------------------------
/libs/lume/src/FileSystem/VMLocation.swift:
--------------------------------------------------------------------------------
```swift
1 | import Foundation
2 |
3 | /// Represents a location where VMs can be stored
4 | struct VMLocation: Codable, Equatable, Sendable {
5 | let name: String
6 | let path: String
7 |
8 | var expandedPath: String {
9 | (path as NSString).expandingTildeInPath
10 | }
11 |
12 | /// Validates the location path exists and is writable
13 | func validate() throws {
14 | let fullPath = expandedPath
15 | var isDir: ObjCBool = false
16 |
17 | if FileManager.default.fileExists(atPath: fullPath, isDirectory: &isDir) {
18 | if !isDir.boolValue {
19 | throw VMLocationError.notADirectory(path: fullPath)
20 | }
21 |
22 | if !FileManager.default.isWritableFile(atPath: fullPath) {
23 | throw VMLocationError.directoryNotWritable(path: fullPath)
24 | }
25 | } else {
26 | // Try to create the directory
27 | do {
28 | try FileManager.default.createDirectory(
29 | atPath: fullPath,
30 | withIntermediateDirectories: true
31 | )
32 | } catch {
33 | throw VMLocationError.directoryCreationFailed(path: fullPath, error: error)
34 | }
35 | }
36 | }
37 | }
38 |
39 | // MARK: - Errors
40 |
41 | enum VMLocationError: Error, LocalizedError {
42 | case notADirectory(path: String)
43 | case directoryNotWritable(path: String)
44 | case directoryCreationFailed(path: String, error: Error)
45 | case locationNotFound(name: String)
46 | case duplicateLocationName(name: String)
47 | case invalidLocationName(name: String)
48 | case defaultLocationCannotBeRemoved(name: String)
49 |
50 | var errorDescription: String? {
51 | switch self {
52 | case .notADirectory(let path):
53 | return "Path is not a directory: \(path)"
54 | case .directoryNotWritable(let path):
55 | return "Directory is not writable: \(path)"
56 | case .directoryCreationFailed(let path, let error):
57 | return "Failed to create directory at \(path): \(error.localizedDescription)"
58 | case .locationNotFound(let name):
59 | return "VM location not found: \(name)"
60 | case .duplicateLocationName(let name):
61 | return "VM location with name '\(name)' already exists"
62 | case .invalidLocationName(let name):
63 | return
64 | "Invalid location name: \(name). Names should be alphanumeric with underscores or dashes."
65 | case .defaultLocationCannotBeRemoved(let name):
66 | return "Cannot remove the default location '\(name)'. Set a new default location first."
67 | }
68 | }
69 | }
70 |
```
--------------------------------------------------------------------------------
/libs/xfce/src/xfce-config/xfce4-session.xml:
--------------------------------------------------------------------------------
```
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <channel name="xfce4-session" version="1.0">
3 | <property name="general" type="empty">
4 | <property name="FailsafeSessionName" type="string" value="Failsafe"/>
5 | <property name="SessionName" type="string" value="Default"/>
6 | <property name="SaveOnExit" type="bool" value="false"/>
7 | </property>
8 | <property name="sessions" type="empty">
9 | <property name="Failsafe" type="empty">
10 | <property name="IsFailsafe" type="bool" value="true"/>
11 | <property name="Count" type="int" value="5"/>
12 | <property name="Client0_Command" type="array">
13 | <value type="string" value="xfwm4"/>
14 | </property>
15 | <property name="Client0_Priority" type="int" value="15"/>
16 | <property name="Client0_PerScreen" type="bool" value="false"/>
17 | <property name="Client1_Command" type="array">
18 | <value type="string" value="xfce4-panel"/>
19 | </property>
20 | <property name="Client1_Priority" type="int" value="25"/>
21 | <property name="Client1_PerScreen" type="bool" value="false"/>
22 | <property name="Client2_Command" type="array">
23 | <value type="string" value="xfdesktop"/>
24 | </property>
25 | <property name="Client2_Priority" type="int" value="35"/>
26 | <property name="Client2_PerScreen" type="bool" value="false"/>
27 | <property name="Client3_Command" type="array">
28 | <value type="string" value="xfsettingsd"/>
29 | </property>
30 | <property name="Client3_Priority" type="int" value="10"/>
31 | <property name="Client3_PerScreen" type="bool" value="false"/>
32 | <property name="Client4_Command" type="array">
33 | <value type="string" value="xfce4-notifyd"/>
34 | </property>
35 | <property name="Client4_Priority" type="int" value="20"/>
36 | <property name="Client4_PerScreen" type="bool" value="false"/>
37 | </property>
38 | </property>
39 | <property name="splash" type="empty">
40 | <property name="Engine" type="string" value=""/>
41 | </property>
42 | <property name="compat" type="empty">
43 | <property name="LaunchGNOME" type="bool" value="false"/>
44 | </property>
45 | <property name="shutdown" type="empty">
46 | <property name="ShowSuspend" type="bool" value="false"/>
47 | <property name="ShowHibernate" type="bool" value="false"/>
48 | <property name="ShowHybridSleep" type="bool" value="false"/>
49 | <property name="ShowSwitchUser" type="bool" value="false"/>
50 | </property>
51 | <property name="screensaver" type="empty">
52 | <property name="enabled" type="bool" value="false"/>
53 | <property name="lock-enabled" type="bool" value="false"/>
54 | </property>
55 | </channel>
56 |
```
--------------------------------------------------------------------------------
/libs/python/agent/tests/conftest.py:
--------------------------------------------------------------------------------
```python
1 | """Pytest configuration and shared fixtures for agent package tests.
2 |
3 | This file contains shared fixtures and configuration for all agent tests.
4 | Following SRP: This file ONLY handles test setup/teardown.
5 | """
6 |
7 | from unittest.mock import AsyncMock, MagicMock, Mock, patch
8 |
9 | import pytest
10 |
11 |
12 | @pytest.fixture
13 | def mock_litellm():
14 | """Mock liteLLM completion calls.
15 |
16 | Use this fixture to avoid making real LLM API calls during tests.
17 | Returns a mock that simulates LLM responses.
18 | """
19 | with patch("litellm.acompletion") as mock_completion:
20 |
21 | async def mock_response(*args, **kwargs):
22 | """Simulate a typical LLM response."""
23 | return {
24 | "id": "chatcmpl-test123",
25 | "object": "chat.completion",
26 | "created": 1234567890,
27 | "model": kwargs.get("model", "anthropic/claude-sonnet-4-5-20250929"),
28 | "choices": [
29 | {
30 | "index": 0,
31 | "message": {
32 | "role": "assistant",
33 | "content": "This is a mocked response for testing.",
34 | },
35 | "finish_reason": "stop",
36 | }
37 | ],
38 | "usage": {
39 | "prompt_tokens": 10,
40 | "completion_tokens": 20,
41 | "total_tokens": 30,
42 | },
43 | }
44 |
45 | mock_completion.side_effect = mock_response
46 | yield mock_completion
47 |
48 |
49 | @pytest.fixture
50 | def mock_computer():
51 | """Mock Computer interface for agent tests.
52 |
53 | Use this fixture to test agent logic without requiring a real Computer instance.
54 | """
55 | computer = AsyncMock()
56 | computer.interface = AsyncMock()
57 | computer.interface.screenshot = AsyncMock(return_value=b"fake_screenshot_data")
58 | computer.interface.left_click = AsyncMock()
59 | computer.interface.type = AsyncMock()
60 | computer.interface.key = AsyncMock()
61 |
62 | # Mock context manager
63 | computer.__aenter__ = AsyncMock(return_value=computer)
64 | computer.__aexit__ = AsyncMock()
65 |
66 | return computer
67 |
68 |
69 | @pytest.fixture
70 | def disable_telemetry(monkeypatch):
71 | """Disable telemetry for tests.
72 |
73 | Use this fixture to ensure no telemetry is sent during tests.
74 | """
75 | monkeypatch.setenv("CUA_TELEMETRY_DISABLED", "1")
76 |
77 |
78 | @pytest.fixture
79 | def sample_messages():
80 | """Provide sample messages for testing.
81 |
82 | Returns a list of messages in the expected format.
83 | """
84 | return [{"role": "user", "content": "Take a screenshot and tell me what you see"}]
85 |
```
--------------------------------------------------------------------------------
/examples/pylume_examples.py:
--------------------------------------------------------------------------------
```python
1 | import asyncio
2 |
3 | from pylume import ImageRef, PyLume, SharedDirectory, VMConfig, VMRunOpts, VMUpdateOpts
4 |
5 |
6 | async def main():
7 | """Example usage of PyLume."""
8 | async with PyLume(port=7777, use_existing_server=False, debug=True) as pylume:
9 |
10 | # Get latest IPSW URL
11 | print("\n=== Getting Latest IPSW URL ===")
12 | url = await pylume.get_latest_ipsw_url()
13 | print("Latest IPSW URL:", url)
14 |
15 | # Create a new VM
16 | print("\n=== Creating a new VM ===")
17 | vm_config = VMConfig(
18 | name="lume-vm-new",
19 | os="macOS",
20 | cpu=2,
21 | memory="4GB",
22 | disk_size="64GB", # type: ignore
23 | display="1024x768",
24 | ipsw="latest",
25 | )
26 | await pylume.create_vm(vm_config)
27 |
28 | # Get latest IPSW URL
29 | print("\n=== Getting Latest IPSW URL ===")
30 | url = await pylume.get_latest_ipsw_url()
31 | print("Latest IPSW URL:", url)
32 |
33 | # List available images
34 | print("\n=== Listing Available Images ===")
35 | images = await pylume.get_images()
36 | print("Available Images:", images)
37 |
38 | # List all VMs to verify creation
39 | print("\n=== Listing All VMs ===")
40 | vms = await pylume.list_vms()
41 | print("VMs:", vms)
42 |
43 | # Get specific VM details
44 | print("\n=== Getting VM Details ===")
45 | vm = await pylume.get_vm("lume-vm")
46 | print("VM Details:", vm)
47 |
48 | # Update VM settings
49 | print("\n=== Updating VM Settings ===")
50 | update_opts = VMUpdateOpts(cpu=8, memory="4GB")
51 | await pylume.update_vm("lume-vm", update_opts)
52 |
53 | # Pull an image
54 | image_ref = ImageRef(
55 | image="macos-sequoia-vanilla", tag="latest", registry="ghcr.io", organization="trycua"
56 | )
57 | await pylume.pull_image(image_ref, name="lume-vm-pulled")
58 |
59 | # Run with shared directory
60 | run_opts = VMRunOpts(
61 | no_display=False, # type: ignore
62 | shared_directories=[ # type: ignore
63 | SharedDirectory(host_path="~/shared", read_only=False) # type: ignore
64 | ],
65 | )
66 | await pylume.run_vm("lume-vm", run_opts)
67 |
68 | # Or simpler:
69 | await pylume.run_vm("lume-vm")
70 |
71 | # Clone VM
72 | print("\n=== Cloning VM ===")
73 | await pylume.clone_vm("lume-vm", "lume-vm-cloned")
74 |
75 | # Stop VM
76 | print("\n=== Stopping VM ===")
77 | await pylume.stop_vm("lume-vm")
78 |
79 | # Delete VM
80 | print("\n=== Deleting VM ===")
81 | await pylume.delete_vm("lume-vm-cloned")
82 |
83 |
84 | if __name__ == "__main__":
85 | asyncio.run(main())
86 |
```
--------------------------------------------------------------------------------
/.github/workflows/link-check.yml:
--------------------------------------------------------------------------------
```yaml
1 | name: Link Checker
2 |
3 | on:
4 | pull_request_target:
5 | branches: [main, master]
6 | push:
7 | branches:
8 | - main
9 | workflow_dispatch:
10 |
11 | jobs:
12 | link-check:
13 | runs-on: ubuntu-latest
14 | steps:
15 | - name: Checkout repository
16 | uses: actions/checkout@v4
17 |
18 | - name: Run Lychee link checker
19 | uses: lycheeverse/lychee-action@v2
20 | id: lychee
21 | with:
22 | # Check all markdown files
23 | args: --verbose --no-progress --max-cache-age 1d --accept 200..=299,403 --exclude '^file://' --exclude 'localhost' --exclude '127\.0\.0\.1' '**/*.md'
24 | # Output results to file for parsing
25 | output: lychee-output.md
26 | # Don't fail the build on broken links (warning mode)
27 | fail: false
28 | env:
29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
30 |
31 | - name: Parse link check results
32 | id: parse-results
33 | if: always()
34 | run: |
35 | # Use lychee exit code: 0 = success, >0 = errors found
36 | EXIT_CODE="${{ steps.lychee.outputs.exit_code }}"
37 |
38 | echo "Exit code: $EXIT_CODE"
39 |
40 | # Show summary if output file exists
41 | if [ -f "lychee-output.md" ]; then
42 | echo "=== Link Check Summary ==="
43 | cat lychee-output.md
44 | fi
45 |
46 | # Set status based on exit code
47 | if [ "$EXIT_CODE" = "0" ]; then
48 | echo "STATUS_ICON=✅" >> $GITHUB_ENV
49 | echo "STATUS_TEXT=All links are working" >> $GITHUB_ENV
50 | echo "COLOR=#36a64f" >> $GITHUB_ENV
51 | elif [ "$EXIT_CODE" = "2" ]; then
52 | echo "STATUS_ICON=❌" >> $GITHUB_ENV
53 | echo "STATUS_TEXT=Link checker failed to run" >> $GITHUB_ENV
54 | echo "COLOR=#dc3545" >> $GITHUB_ENV
55 | else
56 | echo "STATUS_ICON=⚠️" >> $GITHUB_ENV
57 | echo "STATUS_TEXT=Found broken links" >> $GITHUB_ENV
58 | echo "COLOR=#ffa500" >> $GITHUB_ENV
59 | fi
60 |
61 | - name: Send results to Slack
62 | if: always() && github.ref == 'refs/heads/main'
63 | uses: rtCamp/action-slack-notify@v2
64 | env:
65 | SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
66 | SLACK_CHANNEL: ${{ vars.SLACK_CHANNEL }}
67 | SLACK_TITLE: "🔗 Link Check Results"
68 | SLACK_COLOR: ${{ env.COLOR }}
69 | SLACK_MESSAGE: |
70 | *Status:* ${{ env.STATUS_ICON }} ${{ env.STATUS_TEXT }}
71 |
72 | *Branch:* `${{ github.ref_name }}`
73 |
74 | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}${{ github.event.pull_request.number && format('?pr={0}', github.event.pull_request.number) || '' }}|View broken links>
75 |
```
--------------------------------------------------------------------------------
/tests/test_shell_bash.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Shell Command Tests (Bash)
3 | Tests for the run_command method of the Computer interface using bash commands.
4 | Required environment variables:
5 | - CUA_API_KEY: API key for Cua cloud provider
6 | - CUA_CONTAINER_NAME: Name of the container to use
7 | """
8 |
9 | import asyncio
10 | import os
11 | import sys
12 | import traceback
13 | from pathlib import Path
14 |
15 | import pytest
16 |
17 | # Load environment variables from .env file
18 | project_root = Path(__file__).parent.parent
19 | env_file = project_root / ".env"
20 | print(f"Loading environment from: {env_file}")
21 | from dotenv import load_dotenv
22 |
23 | load_dotenv(env_file)
24 |
25 | # Add paths to sys.path if needed
26 | pythonpath = os.environ.get("PYTHONPATH", "")
27 | for path in pythonpath.split(":"):
28 | if path and path not in sys.path:
29 | sys.path.insert(0, path) # Insert at beginning to prioritize
30 | print(f"Added to sys.path: {path}")
31 |
32 | from computer import Computer, VMProviderType
33 |
34 |
35 | @pytest.fixture(scope="session")
36 | async def computer():
37 | """Shared Computer instance for all test cases."""
38 | # Create a remote Linux computer with Cua
39 | computer = Computer(
40 | os_type="linux",
41 | api_key=os.getenv("CUA_API_KEY"),
42 | name=str(os.getenv("CUA_CONTAINER_NAME")),
43 | provider_type=VMProviderType.CLOUD,
44 | )
45 |
46 | try:
47 | await computer.run()
48 | yield computer
49 | finally:
50 | await computer.disconnect()
51 |
52 |
53 | # Sample test cases
54 | @pytest.mark.asyncio(loop_scope="session")
55 | async def test_bash_echo_command(computer):
56 | """Test basic echo command with bash."""
57 | result = await computer.interface.run_command("echo 'Hello World'")
58 |
59 | assert result.stdout.strip() == "Hello World"
60 | assert result.stderr == ""
61 | assert result.returncode == 0
62 |
63 |
64 | @pytest.mark.asyncio(loop_scope="session")
65 | async def test_bash_ls_command(computer):
66 | """Test ls command to list directory contents."""
67 | result = await computer.interface.run_command("ls -la /tmp")
68 |
69 | assert result.returncode == 0
70 | assert result.stderr == ""
71 | assert "total" in result.stdout # ls -la typically starts with "total"
72 | assert "." in result.stdout # Current directory entry
73 | assert ".." in result.stdout # Parent directory entry
74 |
75 |
76 | @pytest.mark.asyncio(loop_scope="session")
77 | async def test_bash_command_with_error(computer):
78 | """Test command that produces an error."""
79 | result = await computer.interface.run_command("ls /nonexistent_directory_12345")
80 |
81 | assert result.returncode != 0
82 | assert result.stdout == ""
83 | assert "No such file or directory" in result.stderr or "cannot access" in result.stderr
84 |
85 |
86 | if __name__ == "__main__":
87 | # Run tests directly
88 | pytest.main([__file__, "-v"])
89 |
```
--------------------------------------------------------------------------------
/tests/shell_cmd.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Shell Command Tests (CMD)
3 | Tests for the run_command method of the Computer interface using cmd.exe commands.
4 | Required environment variables:
5 | - CUA_API_KEY: API key for Cua cloud provider
6 | - CUA_CONTAINER_NAME: Name of the container to use
7 | """
8 |
9 | import asyncio
10 | import os
11 | import sys
12 | import traceback
13 | from pathlib import Path
14 |
15 | import pytest
16 |
17 | # Load environment variables from .env file
18 | project_root = Path(__file__).parent.parent
19 | env_file = project_root / ".env"
20 | print(f"Loading environment from: {env_file}")
21 | from dotenv import load_dotenv
22 |
23 | load_dotenv(env_file)
24 |
25 | # Add paths to sys.path if needed
26 | pythonpath = os.environ.get("PYTHONPATH", "")
27 | for path in pythonpath.split(":"):
28 | if path and path not in sys.path:
29 | sys.path.insert(0, path) # Insert at beginning to prioritize
30 | print(f"Added to sys.path: {path}")
31 |
32 | from computer import Computer, VMProviderType
33 |
34 |
35 | @pytest.fixture(scope="session")
36 | async def computer():
37 | """Shared Computer instance for all test cases."""
38 | # Create a remote Windows computer with Cua
39 | computer = Computer(
40 | os_type="windows",
41 | api_key=os.getenv("CUA_API_KEY"),
42 | name=str(os.getenv("CUA_CONTAINER_NAME")),
43 | provider_type=VMProviderType.CLOUD,
44 | )
45 |
46 | try:
47 | await computer.run()
48 | yield computer
49 | finally:
50 | await computer.disconnect()
51 |
52 |
53 | # Sample test cases
54 | @pytest.mark.asyncio(loop_scope="session")
55 | async def test_cmd_echo_command(computer):
56 | """Test basic echo command with cmd.exe."""
57 | result = await computer.interface.run_command("echo Hello World")
58 |
59 | assert result.stdout.strip() == "Hello World"
60 | assert result.stderr == ""
61 | assert result.returncode == 0
62 |
63 |
64 | @pytest.mark.asyncio(loop_scope="session")
65 | async def test_cmd_dir_command(computer):
66 | """Test dir command to list directory contents."""
67 | result = await computer.interface.run_command("dir C:\\")
68 |
69 | assert result.returncode == 0
70 | assert result.stderr == ""
71 | assert "Directory of C:\\" in result.stdout
72 | assert "bytes" in result.stdout.lower() # dir typically shows file sizes
73 |
74 |
75 | @pytest.mark.asyncio(loop_scope="session")
76 | async def test_cmd_command_with_error(computer):
77 | """Test command that produces an error."""
78 | result = await computer.interface.run_command("dir C:\\nonexistent_directory_12345")
79 |
80 | assert result.returncode != 0
81 | assert result.stdout == ""
82 | assert (
83 | "File Not Found" in result.stderr
84 | or "cannot find the path" in result.stderr
85 | or "The system cannot find" in result.stderr
86 | )
87 |
88 |
89 | if __name__ == "__main__":
90 | # Run tests directly
91 | pytest.main([__file__, "-v"])
92 |
```
--------------------------------------------------------------------------------
/.github/workflows/python-tests.yml:
--------------------------------------------------------------------------------
```yaml
1 | name: Python Unit Tests
2 |
3 | on:
4 | pull_request:
5 | paths:
6 | - "libs/python/**"
7 | - ".github/workflows/python-tests.yml"
8 | push:
9 | branches:
10 | - main
11 | paths:
12 | - "libs/python/**"
13 | - ".github/workflows/python-tests.yml"
14 | workflow_dispatch: # Allow manual trigger
15 |
16 | jobs:
17 | test:
18 | name: Test ${{ matrix.package }}
19 | runs-on: ubuntu-latest
20 |
21 | strategy:
22 | fail-fast: false # Test all packages even if one fails
23 | matrix:
24 | package:
25 | - core
26 | - agent
27 | - computer
28 | - computer-server
29 | - mcp-server
30 | - pylume
31 | - som
32 |
33 | steps:
34 | - name: Checkout code
35 | uses: actions/checkout@v4
36 |
37 | - name: Set up Python
38 | uses: actions/setup-python@v5
39 | with:
40 | python-version: "3.12"
41 |
42 | - name: Install uv
43 | run: |
44 | pip install uv
45 |
46 | - name: Install package and dependencies
47 | run: |
48 | cd libs/python/${{ matrix.package }}
49 | # Install the package in editable mode with dev dependencies
50 | if [ -f pyproject.toml ]; then
51 | uv pip install --system -e .
52 | fi
53 | shell: bash
54 |
55 | - name: Install test dependencies
56 | run: |
57 | # Install test dependencies from root pyproject.toml if tests directory exists
58 | # The root pyproject.toml has package=false, so we install just the dependency group
59 | if [ -d "libs/python/${{ matrix.package }}/tests" ]; then
60 | uv pip install --system --group test
61 | fi
62 | shell: bash
63 |
64 | - name: Run tests
65 | run: |
66 | cd libs/python/${{ matrix.package }}
67 | if [ -d tests ]; then
68 | python -m pytest tests/ -v --tb=short --cov --cov-report=term --cov-report=xml
69 | else
70 | echo "No tests directory found, skipping tests"
71 | fi
72 | shell: bash
73 | env:
74 | CUA_TELEMETRY_DISABLED: "1" # Disable telemetry during tests
75 |
76 | - name: Upload coverage to Codecov
77 | uses: codecov/codecov-action@v4
78 | if: always()
79 | with:
80 | file: ./libs/python/${{ matrix.package }}/coverage.xml
81 | flags: ${{ matrix.package }}
82 | name: codecov-${{ matrix.package }}
83 | fail_ci_if_error: false
84 | continue-on-error: true
85 |
86 | summary:
87 | name: Test Summary
88 | runs-on: ubuntu-latest
89 | needs: test
90 | if: always()
91 |
92 | steps:
93 | - name: Check test results
94 | run: |
95 | if [ "${{ needs.test.result }}" == "failure" ]; then
96 | echo "❌ Some tests failed. Please check the logs above."
97 | exit 1
98 | else
99 | echo "✅ All tests passed!"
100 | fi
101 |
```
--------------------------------------------------------------------------------
/libs/python/agent/agent/loops/base.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Base protocol for async agent configurations
3 | """
4 |
5 | from abc import abstractmethod
6 | from typing import Any, Dict, List, Optional, Protocol, Tuple, Union
7 |
8 | from ..types import AgentCapability
9 |
10 |
11 | class AsyncAgentConfig(Protocol):
12 | """Protocol defining the interface for async agent configurations."""
13 |
14 | @abstractmethod
15 | async def predict_step(
16 | self,
17 | messages: List[Dict[str, Any]],
18 | model: str,
19 | tools: Optional[List[Dict[str, Any]]] = None,
20 | max_retries: Optional[int] = None,
21 | stream: bool = False,
22 | computer_handler=None,
23 | _on_api_start=None,
24 | _on_api_end=None,
25 | _on_usage=None,
26 | _on_screenshot=None,
27 | **generation_config,
28 | ) -> Dict[str, Any]:
29 | """
30 | Predict the next step based on input items.
31 |
32 | Args:
33 | messages: Input items following Responses format (message, function_call, computer_call)
34 | model: Model name to use
35 | tools: Optional list of tool schemas
36 | max_retries: Maximum number of retries for failed API calls
37 | stream: Whether to stream responses
38 | computer_handler: Computer handler instance
39 | _on_api_start: Callback for API start
40 | _on_api_end: Callback for API end
41 | _on_usage: Callback for usage tracking
42 | _on_screenshot: Callback for screenshot events
43 | **generation_config: Additional arguments to pass to the model provider
44 | - api_key: Optional API key for the provider
45 | - api_base: Optional API base URL for the provider
46 |
47 | Returns:
48 | Dictionary with "output" (output items) and "usage" array
49 | """
50 | ...
51 |
52 | @abstractmethod
53 | async def predict_click(
54 | self, model: str, image_b64: str, instruction: str, **generation_config
55 | ) -> Optional[Tuple[int, int]]:
56 | """
57 | Predict click coordinates based on image and instruction.
58 |
59 | Args:
60 | model: Model name to use
61 | image_b64: Base64 encoded image
62 | instruction: Instruction for where to click
63 | **generation_config: Additional arguments to pass to the model provider
64 | - api_key: Optional API key for the provider
65 | - api_base: Optional API base URL for the provider
66 |
67 | Returns:
68 | None or tuple with (x, y) coordinates
69 | """
70 | ...
71 |
72 | @abstractmethod
73 | def get_capabilities(self) -> List[AgentCapability]:
74 | """
75 | Get list of capabilities supported by this agent config.
76 |
77 | Returns:
78 | List of capability strings (e.g., ["step", "click"])
79 | """
80 | ...
81 |
```
--------------------------------------------------------------------------------
/examples/evals/hud_eval_examples.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | hud_eval_examples.py — minimal HUD evaluation runner
3 |
4 | - Auto-discovers .env anywhere up the directory tree (via find_dotenv)
5 | - Requires HUD_API_KEY in the resolved environment
6 | - No Docker/local computer usage
7 | """
8 |
9 | # imports
10 | import asyncio
11 | import logging
12 | import os
13 | import uuid
14 | from pathlib import Path
15 | from pprint import pprint
16 |
17 | from agent import ComputerAgent
18 | from agent.integrations.hud import run_full_dataset
19 | from dotenv import find_dotenv, load_dotenv
20 |
21 | """
22 | Loading env
23 | """
24 |
25 |
26 | def load_env_or_fail() -> None:
27 | # Walk up from CWD / file dir to find nearest .env
28 | env_path = find_dotenv(usecwd=False)
29 | if not env_path:
30 | raise FileNotFoundError(
31 | "❌ .env not found. Place a .env at your repo root (or export HUD_API_KEY)."
32 | )
33 | load_dotenv(env_path, override=True)
34 | if not os.getenv("HUD_API_KEY"):
35 | raise EnvironmentError("❌ HUD_API_KEY is missing in the loaded environment")
36 |
37 |
38 | """
39 | Build Agent Config
40 | - customize agent behavior, tool integration, callbacks, resource management, and more
41 | - https://cua.ai/docs/agent-sdk/agent-loops#parameters
42 | - https://cua.ai/docs/agent-sdk/supported-model-providers
43 | """
44 |
45 |
46 | def build_agent_config() -> dict:
47 |
48 | instruction = "You are a computer-using agent graded by deterministic checkers."
49 |
50 | return {
51 | "model": "openai/computer-use-preview",
52 | "trajectory_dir": str(Path("trajectories")),
53 | "only_n_most_recent_images": 3,
54 | "verbosity": logging.INFO,
55 | "instruction": instruction,
56 | }
57 |
58 |
59 | """
60 | Hud Eval
61 | """
62 |
63 |
64 | async def run_hud_eval() -> None:
65 | # load env and agent config
66 | load_env_or_fail()
67 | agent_config = build_agent_config()
68 |
69 | # Initialize to ensure config is valid (tools, verbosity, etc.)
70 | _ = ComputerAgent(**agent_config)
71 |
72 | job_name = (
73 | f"osworld-test-{str(uuid.uuid4())[:4]}" # job name (each run of your task is a job on hud)
74 | )
75 | print(f"🚀 Running HUD eval: {job_name}")
76 |
77 | """
78 | Customize your hud eval below, check the doc for additional params
79 | - https://cua.ai/docs/agent-sdk/integrations/hud#parameters-1
80 | - recommend low max steps (5-10) for testing, then max 100 for benchmarking
81 | - also select specific tasks to run by using splitting the dataset
82 | """
83 | results = await run_full_dataset(
84 | dataset="ddupont/OSWorld-Tiny-Public",
85 | job_name=job_name,
86 | **agent_config,
87 | max_concurrent=20,
88 | max_steps=50,
89 | # split="train[0:1]"
90 | )
91 |
92 | print(f"\n📊 Job: {job_name}")
93 | print(f"Total results: {len(results)}")
94 | pprint(results[:3])
95 |
96 |
97 | def main() -> None:
98 | logging.basicConfig(level=logging.INFO)
99 | asyncio.run(run_hud_eval())
100 |
101 |
102 | if __name__ == "__main__":
103 | main()
104 |
```
--------------------------------------------------------------------------------
/libs/python/agent/agent/adapters/models/generic.py:
--------------------------------------------------------------------------------
```python
1 | from typing import Any, Dict, List, Optional
2 |
3 | # Hugging Face imports are local to avoid hard dependency at module import
4 | try:
5 | import torch # type: ignore
6 | from transformers import AutoModel, AutoProcessor # type: ignore
7 |
8 | HF_AVAILABLE = True
9 | except Exception:
10 | HF_AVAILABLE = False
11 |
12 |
13 | class GenericHFModel:
14 | """Generic Hugging Face vision-language model handler.
15 | Loads an AutoModelForImageTextToText and AutoProcessor and generates text.
16 | """
17 |
18 | def __init__(
19 | self, model_name: str, device: str = "auto", trust_remote_code: bool = False
20 | ) -> None:
21 | if not HF_AVAILABLE:
22 | raise ImportError(
23 | 'HuggingFace transformers dependencies not found. Install with: pip install "cua-agent[uitars-hf]"'
24 | )
25 | self.model_name = model_name
26 | self.device = device
27 | self.model = None
28 | self.processor = None
29 | self.trust_remote_code = trust_remote_code
30 | self._load()
31 |
32 | def _load(self) -> None:
33 | # Load model
34 | self.model = AutoModel.from_pretrained(
35 | self.model_name,
36 | torch_dtype=torch.float16,
37 | device_map=self.device,
38 | attn_implementation="sdpa",
39 | trust_remote_code=self.trust_remote_code,
40 | )
41 | # Load processor
42 | self.processor = AutoProcessor.from_pretrained(
43 | self.model_name,
44 | min_pixels=3136,
45 | max_pixels=4096 * 2160,
46 | device_map=self.device,
47 | trust_remote_code=self.trust_remote_code,
48 | )
49 |
50 | def generate(self, messages: List[Dict[str, Any]], max_new_tokens: int = 128) -> str:
51 | """Generate text for the given HF-format messages.
52 | messages: [{ role, content: [{type:'text'|'image', text|image}] }]
53 | """
54 | assert self.model is not None and self.processor is not None
55 | # Apply chat template and tokenize
56 | inputs = self.processor.apply_chat_template(
57 | messages,
58 | add_generation_prompt=True,
59 | tokenize=True,
60 | return_dict=True,
61 | return_tensors="pt",
62 | )
63 | # Move inputs to the same device as model
64 | inputs = inputs.to(self.model.device)
65 | # Generate
66 | with torch.no_grad():
67 | generated_ids = self.model.generate(**inputs, max_new_tokens=max_new_tokens)
68 | # Trim prompt tokens from output
69 | generated_ids_trimmed = [
70 | out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
71 | ]
72 | # Decode
73 | output_text = self.processor.batch_decode(
74 | generated_ids_trimmed,
75 | skip_special_tokens=True,
76 | clean_up_tokenization_spaces=False,
77 | )
78 | return output_text[0] if output_text else ""
79 |
```
--------------------------------------------------------------------------------
/docs/content/docs/computer-sdk/computer-ui.mdx:
--------------------------------------------------------------------------------
```markdown
1 | ---
2 | title: Computer UI
3 | ---
4 |
5 | <Callout type="warn" title="Deprecated">
6 | The Computer UI is deprecated and will be replaced with a revamped playground experience soon. We
7 | recommend using VNC or Screen Sharing for precise control of the computer instead.
8 | </Callout>
9 |
10 | The computer module includes a Gradio UI for creating and sharing demonstration data. We make it easy for people to build community datasets for better computer use models with an upload to Huggingface feature.
11 |
12 | ```bash
13 | # Install with UI support
14 | pip install "cua-computer[ui]"
15 | ```
16 |
17 | <Callout title="Note">
18 | For precise control of the computer, we recommend using VNC or Screen Sharing instead of the
19 | Computer Gradio UI.
20 | </Callout>
21 |
22 | ### Building and Sharing Demonstrations with Huggingface
23 |
24 | Follow these steps to contribute your own demonstrations:
25 |
26 | #### 1. Set up Huggingface Access
27 |
28 | Set your HF_TOKEN in a .env file or in your environment variables:
29 |
30 | ```bash
31 | # In .env file
32 | HF_TOKEN=your_huggingface_token
33 | ```
34 |
35 | #### 2. Launch the Computer UI
36 |
37 | ```python
38 | # launch_ui.py
39 | from computer.ui.gradio.app import create_gradio_ui
40 | from dotenv import load_dotenv
41 | load_dotenv('.env')
42 |
43 | app = create_gradio_ui()
44 | app.launch(share=False)
45 | ```
46 |
47 | For examples, see [Computer UI Examples](https://github.com/trycua/cua/tree/main/examples/computer_ui_examples.py)
48 |
49 | #### 3. Record Your Tasks
50 |
51 | <details open>
52 | <summary>View demonstration video</summary>
53 | <video
54 | src="https://github.com/user-attachments/assets/de3c3477-62fe-413c-998d-4063e48de176"
55 | controls
56 | width="600"
57 | ></video>
58 | </details>
59 |
60 | Record yourself performing various computer tasks using the UI.
61 |
62 | #### 4. Save Your Demonstrations
63 |
64 | <details open>
65 | <summary>View demonstration video</summary>
66 | <video
67 | src="https://github.com/user-attachments/assets/5ad1df37-026a-457f-8b49-922ae805faef"
68 | controls
69 | width="600"
70 | ></video>
71 | </details>
72 |
73 | Save each task by picking a descriptive name and adding relevant tags (e.g., "office", "web-browsing", "coding").
74 |
75 | #### 5. Record Additional Demonstrations
76 |
77 | Repeat steps 3 and 4 until you have a good amount of demonstrations covering different tasks and scenarios.
78 |
79 | #### 6. Upload to Huggingface
80 |
81 | <details open>
82 | <summary>View demonstration video</summary>
83 | <video
84 | src="https://github.com/user-attachments/assets/c586d460-3877-4b5f-a736-3248886d2134"
85 | controls
86 | width="600"
87 | ></video>
88 | </details>
89 |
90 | Upload your dataset to Huggingface by:
91 |
92 | - Naming it as `{your_username}/{dataset_name}`
93 | - Choosing public or private visibility
94 | - Optionally selecting specific tags to upload only tasks with certain tags
95 |
96 | #### Examples and Resources
97 |
98 | - Example Dataset: [ddupont/test-dataset](https://huggingface.co/datasets/ddupont/test-dataset)
99 | - Find Community Datasets: 🔍 [Browse CUA Datasets on Huggingface](https://huggingface.co/datasets?other=cua)
100 |
```
--------------------------------------------------------------------------------
/libs/python/agent/agent/adapters/models/qwen2_5_vl.py:
--------------------------------------------------------------------------------
```python
1 | from typing import Any, Dict, List, Optional
2 |
3 | # Hugging Face imports are local to avoid hard dependency at module import
4 | try:
5 | import torch # type: ignore
6 | from transformers import AutoModelForImageTextToText, AutoProcessor # type: ignore
7 |
8 | HF_AVAILABLE = True
9 | except Exception:
10 | HF_AVAILABLE = False
11 |
12 |
13 | class Qwen2_5_VLModel:
14 | """Qwen2.5-VL Hugging Face vision-language model handler.
15 | Loads an AutoModelForImageTextToText and AutoProcessor and generates text.
16 | """
17 |
18 | def __init__(
19 | self, model_name: str, device: str = "auto", trust_remote_code: bool = False
20 | ) -> None:
21 | if not HF_AVAILABLE:
22 | raise ImportError(
23 | 'HuggingFace transformers dependencies not found. Install with: pip install "cua-agent[uitars-hf]"'
24 | )
25 | self.model_name = model_name
26 | self.device = device
27 | self.model = None
28 | self.processor = None
29 | self.trust_remote_code = trust_remote_code
30 | self._load()
31 |
32 | def _load(self) -> None:
33 | # Load model
34 | self.model = AutoModelForImageTextToText.from_pretrained(
35 | self.model_name,
36 | torch_dtype=torch.bfloat16,
37 | device_map=self.device,
38 | attn_implementation="sdpa",
39 | trust_remote_code=self.trust_remote_code,
40 | )
41 | # Load processor
42 | self.processor = AutoProcessor.from_pretrained(
43 | self.model_name,
44 | min_pixels=3136,
45 | max_pixels=4096 * 2160,
46 | device_map=self.device,
47 | trust_remote_code=self.trust_remote_code,
48 | )
49 |
50 | def generate(self, messages: List[Dict[str, Any]], max_new_tokens: int = 128) -> str:
51 | """Generate text for the given HF-format messages.
52 | messages: [{ role, content: [{type:'text'|'image', text|image}] }]
53 | """
54 | assert self.model is not None and self.processor is not None
55 | # Apply chat template and tokenize
56 | inputs = self.processor.apply_chat_template(
57 | messages,
58 | add_generation_prompt=True,
59 | tokenize=True,
60 | return_dict=True,
61 | return_tensors="pt",
62 | )
63 | # Move inputs to the same device as model
64 | inputs = inputs.to(self.model.device)
65 | # Generate
66 | with torch.no_grad():
67 | generated_ids = self.model.generate(**inputs, max_new_tokens=max_new_tokens)
68 | # Trim prompt tokens from output
69 | generated_ids_trimmed = [
70 | out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
71 | ]
72 | # Decode
73 | output_text = self.processor.batch_decode(
74 | generated_ids_trimmed,
75 | skip_special_tokens=True,
76 | clean_up_tokenization_spaces=False,
77 | )
78 | return output_text[0] if output_text else ""
79 |
```