This is page 14 of 21. Use http://codebase.md/oraios/serena?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .devcontainer
│ └── devcontainer.json
├── .dockerignore
├── .env.example
├── .github
│ ├── FUNDING.yml
│ ├── ISSUE_TEMPLATE
│ │ ├── config.yml
│ │ ├── feature_request.md
│ │ └── issue--bug--performance-problem--question-.md
│ └── workflows
│ ├── codespell.yml
│ ├── docker.yml
│ ├── docs.yaml
│ ├── junie.yml
│ ├── publish.yml
│ └── pytest.yml
├── .gitignore
├── .serena
│ ├── .gitignore
│ ├── memories
│ │ ├── adding_new_language_support_guide.md
│ │ ├── serena_core_concepts_and_architecture.md
│ │ ├── serena_repository_structure.md
│ │ └── suggested_commands.md
│ └── project.yml
├── .vscode
│ └── settings.json
├── CHANGELOG.md
├── CLAUDE.md
├── compose.yaml
├── CONTRIBUTING.md
├── docker_build_and_run.sh
├── DOCKER.md
├── Dockerfile
├── docs
│ ├── _config.yml
│ ├── _static
│ │ └── images
│ │ └── jetbrains-marketplace-button.png
│ ├── .gitignore
│ ├── 01-about
│ │ ├── 000_intro.md
│ │ ├── 010_llm-integration.md
│ │ ├── 020_programming-languages.md
│ │ ├── 030_serena-in-action.md
│ │ ├── 035_tools.md
│ │ ├── 040_comparison-to-other-agents.md
│ │ └── 050_acknowledgements.md
│ ├── 02-usage
│ │ ├── 000_intro.md
│ │ ├── 010_prerequisites.md
│ │ ├── 020_running.md
│ │ ├── 025_jetbrains_plugin.md
│ │ ├── 030_clients.md
│ │ ├── 040_workflow.md
│ │ ├── 050_configuration.md
│ │ ├── 060_dashboard.md
│ │ ├── 070_security.md
│ │ └── 999_additional-usage.md
│ ├── 03-special-guides
│ │ ├── 000_intro.md
│ │ ├── custom_agent.md
│ │ ├── groovy_setup_guide_for_serena.md
│ │ ├── scala_setup_guide_for_serena.md
│ │ └── serena_on_chatgpt.md
│ ├── autogen_rst.py
│ ├── create_toc.py
│ └── index.md
├── flake.lock
├── flake.nix
├── lessons_learned.md
├── LICENSE
├── llms-install.md
├── pyproject.toml
├── README.md
├── repo_dir_sync.py
├── resources
│ ├── jetbrains-marketplace-button.cdr
│ ├── serena-icons.cdr
│ ├── serena-logo-dark-mode.svg
│ ├── serena-logo.cdr
│ ├── serena-logo.svg
│ └── vscode_sponsor_logo.png
├── roadmap.md
├── scripts
│ ├── agno_agent.py
│ ├── demo_run_tools.py
│ ├── gen_prompt_factory.py
│ ├── mcp_server.py
│ ├── print_mode_context_options.py
│ ├── print_tool_overview.py
│ └── profile_tool_call.py
├── src
│ ├── interprompt
│ │ ├── __init__.py
│ │ ├── .syncCommitId.remote
│ │ ├── .syncCommitId.this
│ │ ├── jinja_template.py
│ │ ├── multilang_prompt.py
│ │ ├── prompt_factory.py
│ │ └── util
│ │ ├── __init__.py
│ │ └── class_decorators.py
│ ├── README.md
│ ├── serena
│ │ ├── __init__.py
│ │ ├── agent.py
│ │ ├── agno.py
│ │ ├── analytics.py
│ │ ├── cli.py
│ │ ├── code_editor.py
│ │ ├── config
│ │ │ ├── __init__.py
│ │ │ ├── context_mode.py
│ │ │ └── serena_config.py
│ │ ├── constants.py
│ │ ├── dashboard.py
│ │ ├── generated
│ │ │ └── generated_prompt_factory.py
│ │ ├── gui_log_viewer.py
│ │ ├── ls_manager.py
│ │ ├── mcp.py
│ │ ├── project.py
│ │ ├── prompt_factory.py
│ │ ├── resources
│ │ │ ├── config
│ │ │ │ ├── contexts
│ │ │ │ │ ├── agent.yml
│ │ │ │ │ ├── chatgpt.yml
│ │ │ │ │ ├── claude-code.yml
│ │ │ │ │ ├── codex.yml
│ │ │ │ │ ├── context.template.yml
│ │ │ │ │ ├── desktop-app.yml
│ │ │ │ │ ├── ide.yml
│ │ │ │ │ └── oaicompat-agent.yml
│ │ │ │ ├── internal_modes
│ │ │ │ │ └── jetbrains.yml
│ │ │ │ ├── modes
│ │ │ │ │ ├── editing.yml
│ │ │ │ │ ├── interactive.yml
│ │ │ │ │ ├── mode.template.yml
│ │ │ │ │ ├── no-memories.yml
│ │ │ │ │ ├── no-onboarding.yml
│ │ │ │ │ ├── onboarding.yml
│ │ │ │ │ ├── one-shot.yml
│ │ │ │ │ └── planning.yml
│ │ │ │ └── prompt_templates
│ │ │ │ ├── simple_tool_outputs.yml
│ │ │ │ └── system_prompt.yml
│ │ │ ├── dashboard
│ │ │ │ ├── dashboard.css
│ │ │ │ ├── dashboard.js
│ │ │ │ ├── index.html
│ │ │ │ ├── jquery.min.js
│ │ │ │ ├── serena-icon-16.png
│ │ │ │ ├── serena-icon-32.png
│ │ │ │ ├── serena-icon-48.png
│ │ │ │ ├── serena-logo-dark-mode.svg
│ │ │ │ ├── serena-logo.svg
│ │ │ │ ├── serena-logs-dark-mode.png
│ │ │ │ └── serena-logs.png
│ │ │ ├── project.template.yml
│ │ │ └── serena_config.template.yml
│ │ ├── symbol.py
│ │ ├── task_executor.py
│ │ ├── text_utils.py
│ │ ├── tools
│ │ │ ├── __init__.py
│ │ │ ├── cmd_tools.py
│ │ │ ├── config_tools.py
│ │ │ ├── file_tools.py
│ │ │ ├── jetbrains_plugin_client.py
│ │ │ ├── jetbrains_tools.py
│ │ │ ├── memory_tools.py
│ │ │ ├── symbol_tools.py
│ │ │ ├── tools_base.py
│ │ │ └── workflow_tools.py
│ │ └── util
│ │ ├── class_decorators.py
│ │ ├── cli_util.py
│ │ ├── exception.py
│ │ ├── file_system.py
│ │ ├── general.py
│ │ ├── git.py
│ │ ├── gui.py
│ │ ├── inspection.py
│ │ ├── logging.py
│ │ ├── shell.py
│ │ └── thread.py
│ └── solidlsp
│ ├── __init__.py
│ ├── .gitignore
│ ├── language_servers
│ │ ├── al_language_server.py
│ │ ├── bash_language_server.py
│ │ ├── clangd_language_server.py
│ │ ├── clojure_lsp.py
│ │ ├── common.py
│ │ ├── csharp_language_server.py
│ │ ├── dart_language_server.py
│ │ ├── eclipse_jdtls.py
│ │ ├── elixir_tools
│ │ │ ├── __init__.py
│ │ │ ├── elixir_tools.py
│ │ │ └── README.md
│ │ ├── elm_language_server.py
│ │ ├── erlang_language_server.py
│ │ ├── fortran_language_server.py
│ │ ├── fsharp_language_server.py
│ │ ├── gopls.py
│ │ ├── groovy_language_server.py
│ │ ├── haskell_language_server.py
│ │ ├── intelephense.py
│ │ ├── jedi_server.py
│ │ ├── julia_server.py
│ │ ├── kotlin_language_server.py
│ │ ├── lua_ls.py
│ │ ├── marksman.py
│ │ ├── matlab_language_server.py
│ │ ├── nixd_ls.py
│ │ ├── omnisharp
│ │ │ ├── initialize_params.json
│ │ │ ├── runtime_dependencies.json
│ │ │ └── workspace_did_change_configuration.json
│ │ ├── omnisharp.py
│ │ ├── pascal_server.py
│ │ ├── perl_language_server.py
│ │ ├── powershell_language_server.py
│ │ ├── pyright_server.py
│ │ ├── r_language_server.py
│ │ ├── regal_server.py
│ │ ├── ruby_lsp.py
│ │ ├── rust_analyzer.py
│ │ ├── scala_language_server.py
│ │ ├── solargraph.py
│ │ ├── sourcekit_lsp.py
│ │ ├── taplo_server.py
│ │ ├── terraform_ls.py
│ │ ├── typescript_language_server.py
│ │ ├── vts_language_server.py
│ │ ├── vue_language_server.py
│ │ ├── yaml_language_server.py
│ │ └── zls.py
│ ├── ls_config.py
│ ├── ls_exceptions.py
│ ├── ls_handler.py
│ ├── ls_request.py
│ ├── ls_types.py
│ ├── ls_utils.py
│ ├── ls.py
│ ├── lsp_protocol_handler
│ │ ├── lsp_constants.py
│ │ ├── lsp_requests.py
│ │ ├── lsp_types.py
│ │ └── server.py
│ ├── settings.py
│ └── util
│ ├── cache.py
│ ├── subprocess_util.py
│ └── zip.py
├── sync.py
├── test
│ ├── __init__.py
│ ├── conftest.py
│ ├── resources
│ │ └── repos
│ │ ├── al
│ │ │ └── test_repo
│ │ │ ├── app.json
│ │ │ └── src
│ │ │ ├── Codeunits
│ │ │ │ ├── CustomerMgt.Codeunit.al
│ │ │ │ └── PaymentProcessorImpl.Codeunit.al
│ │ │ ├── Enums
│ │ │ │ └── CustomerType.Enum.al
│ │ │ ├── Interfaces
│ │ │ │ └── IPaymentProcessor.Interface.al
│ │ │ ├── Pages
│ │ │ │ ├── CustomerCard.Page.al
│ │ │ │ └── CustomerList.Page.al
│ │ │ ├── TableExtensions
│ │ │ │ └── Item.TableExt.al
│ │ │ └── Tables
│ │ │ └── Customer.Table.al
│ │ ├── bash
│ │ │ └── test_repo
│ │ │ ├── config.sh
│ │ │ ├── main.sh
│ │ │ └── utils.sh
│ │ ├── clojure
│ │ │ └── test_repo
│ │ │ ├── deps.edn
│ │ │ └── src
│ │ │ └── test_app
│ │ │ ├── core.clj
│ │ │ └── utils.clj
│ │ ├── csharp
│ │ │ └── test_repo
│ │ │ ├── .gitignore
│ │ │ ├── Models
│ │ │ │ └── Person.cs
│ │ │ ├── Program.cs
│ │ │ ├── serena.sln
│ │ │ └── TestProject.csproj
│ │ ├── dart
│ │ │ └── test_repo
│ │ │ ├── .gitignore
│ │ │ ├── lib
│ │ │ │ ├── helper.dart
│ │ │ │ ├── main.dart
│ │ │ │ └── models.dart
│ │ │ └── pubspec.yaml
│ │ ├── elixir
│ │ │ └── test_repo
│ │ │ ├── .gitignore
│ │ │ ├── lib
│ │ │ │ ├── examples.ex
│ │ │ │ ├── ignored_dir
│ │ │ │ │ └── ignored_module.ex
│ │ │ │ ├── models.ex
│ │ │ │ ├── services.ex
│ │ │ │ ├── test_repo.ex
│ │ │ │ └── utils.ex
│ │ │ ├── mix.exs
│ │ │ ├── mix.lock
│ │ │ ├── scripts
│ │ │ │ └── build_script.ex
│ │ │ └── test
│ │ │ ├── models_test.exs
│ │ │ └── test_repo_test.exs
│ │ ├── elm
│ │ │ └── test_repo
│ │ │ ├── elm.json
│ │ │ ├── Main.elm
│ │ │ └── Utils.elm
│ │ ├── erlang
│ │ │ └── test_repo
│ │ │ ├── hello.erl
│ │ │ ├── ignored_dir
│ │ │ │ └── ignored_module.erl
│ │ │ ├── include
│ │ │ │ ├── records.hrl
│ │ │ │ └── types.hrl
│ │ │ ├── math_utils.erl
│ │ │ ├── rebar.config
│ │ │ ├── src
│ │ │ │ ├── app.erl
│ │ │ │ ├── models.erl
│ │ │ │ ├── services.erl
│ │ │ │ └── utils.erl
│ │ │ └── test
│ │ │ ├── models_tests.erl
│ │ │ └── utils_tests.erl
│ │ ├── fortran
│ │ │ └── test_repo
│ │ │ ├── main.f90
│ │ │ └── modules
│ │ │ ├── geometry.f90
│ │ │ └── math_utils.f90
│ │ ├── fsharp
│ │ │ └── test_repo
│ │ │ ├── .gitignore
│ │ │ ├── Calculator.fs
│ │ │ ├── Models
│ │ │ │ └── Person.fs
│ │ │ ├── Program.fs
│ │ │ ├── README.md
│ │ │ └── TestProject.fsproj
│ │ ├── go
│ │ │ └── test_repo
│ │ │ └── main.go
│ │ ├── groovy
│ │ │ └── test_repo
│ │ │ ├── .gitignore
│ │ │ ├── build.gradle
│ │ │ └── src
│ │ │ └── main
│ │ │ └── groovy
│ │ │ └── com
│ │ │ └── example
│ │ │ ├── Main.groovy
│ │ │ ├── Model.groovy
│ │ │ ├── ModelUser.groovy
│ │ │ └── Utils.groovy
│ │ ├── haskell
│ │ │ └── test_repo
│ │ │ ├── app
│ │ │ │ └── Main.hs
│ │ │ ├── haskell-test-repo.cabal
│ │ │ ├── package.yaml
│ │ │ ├── src
│ │ │ │ ├── Calculator.hs
│ │ │ │ └── Helper.hs
│ │ │ └── stack.yaml
│ │ ├── java
│ │ │ └── test_repo
│ │ │ ├── pom.xml
│ │ │ └── src
│ │ │ └── main
│ │ │ └── java
│ │ │ └── test_repo
│ │ │ ├── Main.java
│ │ │ ├── Model.java
│ │ │ ├── ModelUser.java
│ │ │ └── Utils.java
│ │ ├── julia
│ │ │ └── test_repo
│ │ │ ├── lib
│ │ │ │ └── helper.jl
│ │ │ └── main.jl
│ │ ├── kotlin
│ │ │ └── test_repo
│ │ │ ├── .gitignore
│ │ │ ├── build.gradle.kts
│ │ │ └── src
│ │ │ └── main
│ │ │ └── kotlin
│ │ │ └── test_repo
│ │ │ ├── Main.kt
│ │ │ ├── Model.kt
│ │ │ ├── ModelUser.kt
│ │ │ └── Utils.kt
│ │ ├── lua
│ │ │ └── test_repo
│ │ │ ├── .gitignore
│ │ │ ├── main.lua
│ │ │ ├── src
│ │ │ │ ├── calculator.lua
│ │ │ │ └── utils.lua
│ │ │ └── tests
│ │ │ └── test_calculator.lua
│ │ ├── markdown
│ │ │ └── test_repo
│ │ │ ├── api.md
│ │ │ ├── CONTRIBUTING.md
│ │ │ ├── guide.md
│ │ │ └── README.md
│ │ ├── matlab
│ │ │ └── test_repo
│ │ │ ├── Calculator.m
│ │ │ └── main.m
│ │ ├── nix
│ │ │ └── test_repo
│ │ │ ├── .gitignore
│ │ │ ├── default.nix
│ │ │ ├── flake.nix
│ │ │ ├── lib
│ │ │ │ └── utils.nix
│ │ │ ├── modules
│ │ │ │ └── example.nix
│ │ │ └── scripts
│ │ │ └── hello.sh
│ │ ├── pascal
│ │ │ └── test_repo
│ │ │ ├── .gitignore
│ │ │ ├── lib
│ │ │ │ └── helper.pas
│ │ │ └── main.pas
│ │ ├── perl
│ │ │ └── test_repo
│ │ │ ├── helper.pl
│ │ │ └── main.pl
│ │ ├── php
│ │ │ └── test_repo
│ │ │ ├── helper.php
│ │ │ ├── index.php
│ │ │ └── simple_var.php
│ │ ├── powershell
│ │ │ └── test_repo
│ │ │ ├── main.ps1
│ │ │ ├── PowerShellEditorServices.json
│ │ │ └── utils.ps1
│ │ ├── python
│ │ │ └── test_repo
│ │ │ ├── .gitignore
│ │ │ ├── custom_test
│ │ │ │ ├── __init__.py
│ │ │ │ └── advanced_features.py
│ │ │ ├── examples
│ │ │ │ ├── __init__.py
│ │ │ │ └── user_management.py
│ │ │ ├── ignore_this_dir_with_postfix
│ │ │ │ └── ignored_module.py
│ │ │ ├── scripts
│ │ │ │ ├── __init__.py
│ │ │ │ └── run_app.py
│ │ │ └── test_repo
│ │ │ ├── __init__.py
│ │ │ ├── complex_types.py
│ │ │ ├── models.py
│ │ │ ├── name_collisions.py
│ │ │ ├── nested_base.py
│ │ │ ├── nested.py
│ │ │ ├── overloaded.py
│ │ │ ├── services.py
│ │ │ ├── utils.py
│ │ │ └── variables.py
│ │ ├── r
│ │ │ └── test_repo
│ │ │ ├── .Rbuildignore
│ │ │ ├── DESCRIPTION
│ │ │ ├── examples
│ │ │ │ └── analysis.R
│ │ │ ├── NAMESPACE
│ │ │ └── R
│ │ │ ├── models.R
│ │ │ └── utils.R
│ │ ├── rego
│ │ │ └── test_repo
│ │ │ ├── policies
│ │ │ │ ├── authz.rego
│ │ │ │ └── validation.rego
│ │ │ └── utils
│ │ │ └── helpers.rego
│ │ ├── ruby
│ │ │ └── test_repo
│ │ │ ├── .solargraph.yml
│ │ │ ├── examples
│ │ │ │ └── user_management.rb
│ │ │ ├── lib.rb
│ │ │ ├── main.rb
│ │ │ ├── models.rb
│ │ │ ├── nested.rb
│ │ │ ├── services.rb
│ │ │ └── variables.rb
│ │ ├── rust
│ │ │ ├── test_repo
│ │ │ │ ├── Cargo.lock
│ │ │ │ ├── Cargo.toml
│ │ │ │ └── src
│ │ │ │ ├── lib.rs
│ │ │ │ └── main.rs
│ │ │ └── test_repo_2024
│ │ │ ├── Cargo.lock
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── lib.rs
│ │ │ └── main.rs
│ │ ├── scala
│ │ │ ├── build.sbt
│ │ │ ├── project
│ │ │ │ ├── build.properties
│ │ │ │ ├── metals.sbt
│ │ │ │ └── plugins.sbt
│ │ │ └── src
│ │ │ └── main
│ │ │ └── scala
│ │ │ └── com
│ │ │ └── example
│ │ │ ├── Main.scala
│ │ │ └── Utils.scala
│ │ ├── swift
│ │ │ └── test_repo
│ │ │ ├── Package.swift
│ │ │ └── src
│ │ │ ├── main.swift
│ │ │ └── utils.swift
│ │ ├── terraform
│ │ │ └── test_repo
│ │ │ ├── data.tf
│ │ │ ├── main.tf
│ │ │ ├── outputs.tf
│ │ │ └── variables.tf
│ │ ├── toml
│ │ │ └── test_repo
│ │ │ ├── Cargo.toml
│ │ │ ├── config.toml
│ │ │ └── pyproject.toml
│ │ ├── typescript
│ │ │ └── test_repo
│ │ │ ├── .serena
│ │ │ │ └── project.yml
│ │ │ ├── index.ts
│ │ │ ├── tsconfig.json
│ │ │ ├── use_helper.ts
│ │ │ └── ws_manager.js
│ │ ├── vue
│ │ │ └── test_repo
│ │ │ ├── .gitignore
│ │ │ ├── index.html
│ │ │ ├── package.json
│ │ │ ├── src
│ │ │ │ ├── App.vue
│ │ │ │ ├── components
│ │ │ │ │ ├── CalculatorButton.vue
│ │ │ │ │ ├── CalculatorDisplay.vue
│ │ │ │ │ └── CalculatorInput.vue
│ │ │ │ ├── composables
│ │ │ │ │ ├── useFormatter.ts
│ │ │ │ │ └── useTheme.ts
│ │ │ │ ├── main.ts
│ │ │ │ ├── stores
│ │ │ │ │ └── calculator.ts
│ │ │ │ └── types
│ │ │ │ └── index.ts
│ │ │ ├── tsconfig.json
│ │ │ ├── tsconfig.node.json
│ │ │ └── vite.config.ts
│ │ ├── yaml
│ │ │ └── test_repo
│ │ │ ├── config.yaml
│ │ │ ├── data.yaml
│ │ │ └── services.yml
│ │ └── zig
│ │ └── test_repo
│ │ ├── .gitignore
│ │ ├── build.zig
│ │ ├── src
│ │ │ ├── calculator.zig
│ │ │ ├── main.zig
│ │ │ └── math_utils.zig
│ │ └── zls.json
│ ├── serena
│ │ ├── __init__.py
│ │ ├── __snapshots__
│ │ │ └── test_symbol_editing.ambr
│ │ ├── config
│ │ │ ├── __init__.py
│ │ │ └── test_serena_config.py
│ │ ├── test_cli_project_commands.py
│ │ ├── test_edit_marker.py
│ │ ├── test_mcp.py
│ │ ├── test_serena_agent.py
│ │ ├── test_symbol_editing.py
│ │ ├── test_symbol.py
│ │ ├── test_task_executor.py
│ │ ├── test_text_utils.py
│ │ ├── test_tool_parameter_types.py
│ │ └── util
│ │ ├── test_exception.py
│ │ └── test_file_system.py
│ └── solidlsp
│ ├── al
│ │ └── test_al_basic.py
│ ├── bash
│ │ ├── __init__.py
│ │ └── test_bash_basic.py
│ ├── clojure
│ │ ├── __init__.py
│ │ └── test_clojure_basic.py
│ ├── csharp
│ │ └── test_csharp_basic.py
│ ├── dart
│ │ ├── __init__.py
│ │ └── test_dart_basic.py
│ ├── elixir
│ │ ├── __init__.py
│ │ ├── conftest.py
│ │ ├── test_elixir_basic.py
│ │ ├── test_elixir_ignored_dirs.py
│ │ ├── test_elixir_integration.py
│ │ └── test_elixir_symbol_retrieval.py
│ ├── elm
│ │ └── test_elm_basic.py
│ ├── erlang
│ │ ├── __init__.py
│ │ ├── conftest.py
│ │ ├── test_erlang_basic.py
│ │ ├── test_erlang_ignored_dirs.py
│ │ └── test_erlang_symbol_retrieval.py
│ ├── fortran
│ │ ├── __init__.py
│ │ └── test_fortran_basic.py
│ ├── fsharp
│ │ └── test_fsharp_basic.py
│ ├── go
│ │ └── test_go_basic.py
│ ├── groovy
│ │ └── test_groovy_basic.py
│ ├── haskell
│ │ ├── __init__.py
│ │ └── test_haskell_basic.py
│ ├── java
│ │ └── test_java_basic.py
│ ├── julia
│ │ └── test_julia_basic.py
│ ├── kotlin
│ │ └── test_kotlin_basic.py
│ ├── lua
│ │ └── test_lua_basic.py
│ ├── markdown
│ │ ├── __init__.py
│ │ └── test_markdown_basic.py
│ ├── matlab
│ │ ├── __init__.py
│ │ └── test_matlab_basic.py
│ ├── nix
│ │ └── test_nix_basic.py
│ ├── pascal
│ │ ├── __init__.py
│ │ └── test_pascal_basic.py
│ ├── perl
│ │ └── test_perl_basic.py
│ ├── php
│ │ └── test_php_basic.py
│ ├── powershell
│ │ ├── __init__.py
│ │ └── test_powershell_basic.py
│ ├── python
│ │ ├── test_python_basic.py
│ │ ├── test_retrieval_with_ignored_dirs.py
│ │ └── test_symbol_retrieval.py
│ ├── r
│ │ ├── __init__.py
│ │ └── test_r_basic.py
│ ├── rego
│ │ └── test_rego_basic.py
│ ├── ruby
│ │ ├── test_ruby_basic.py
│ │ └── test_ruby_symbol_retrieval.py
│ ├── rust
│ │ ├── test_rust_2024_edition.py
│ │ ├── test_rust_analyzer_detection.py
│ │ └── test_rust_basic.py
│ ├── scala
│ │ └── test_scala_language_server.py
│ ├── swift
│ │ └── test_swift_basic.py
│ ├── terraform
│ │ └── test_terraform_basic.py
│ ├── test_lsp_protocol_handler_server.py
│ ├── toml
│ │ ├── __init__.py
│ │ ├── test_toml_basic.py
│ │ ├── test_toml_edge_cases.py
│ │ ├── test_toml_ignored_dirs.py
│ │ └── test_toml_symbol_retrieval.py
│ ├── typescript
│ │ └── test_typescript_basic.py
│ ├── util
│ │ └── test_zip.py
│ ├── vue
│ │ ├── __init__.py
│ │ ├── test_vue_basic.py
│ │ ├── test_vue_error_cases.py
│ │ ├── test_vue_rename.py
│ │ └── test_vue_symbol_retrieval.py
│ ├── yaml_ls
│ │ ├── __init__.py
│ │ └── test_yaml_basic.py
│ └── zig
│ └── test_zig_basic.py
└── uv.lock
```
# Files
--------------------------------------------------------------------------------
/src/serena/dashboard.py:
--------------------------------------------------------------------------------
```python
1 | import os
2 | import socket
3 | import threading
4 | from collections.abc import Callable
5 | from typing import TYPE_CHECKING, Any, Self
6 |
7 | from flask import Flask, Response, request, send_from_directory
8 | from pydantic import BaseModel
9 | from sensai.util import logging
10 |
11 | from serena.analytics import ToolUsageStats
12 | from serena.config.serena_config import LanguageBackend
13 | from serena.constants import SERENA_DASHBOARD_DIR
14 | from serena.task_executor import TaskExecutor
15 | from serena.util.logging import MemoryLogHandler
16 |
17 | if TYPE_CHECKING:
18 | from serena.agent import SerenaAgent
19 |
20 | log = logging.getLogger(__name__)
21 |
22 | # disable Werkzeug's logging to avoid cluttering the output
23 | logging.getLogger("werkzeug").setLevel(logging.WARNING)
24 |
25 |
26 | class RequestLog(BaseModel):
27 | start_idx: int = 0
28 |
29 |
30 | class ResponseLog(BaseModel):
31 | messages: list[str]
32 | max_idx: int
33 | active_project: str | None = None
34 |
35 |
36 | class ResponseToolNames(BaseModel):
37 | tool_names: list[str]
38 |
39 |
40 | class ResponseToolStats(BaseModel):
41 | stats: dict[str, dict[str, int]]
42 |
43 |
44 | class ResponseConfigOverview(BaseModel):
45 | active_project: dict[str, str | None]
46 | context: dict[str, str]
47 | modes: list[dict[str, str]]
48 | active_tools: list[str]
49 | tool_stats_summary: dict[str, dict[str, int]]
50 | registered_projects: list[dict[str, str | bool]]
51 | available_tools: list[dict[str, str | bool]]
52 | available_modes: list[dict[str, str | bool]]
53 | available_contexts: list[dict[str, str | bool]]
54 | available_memories: list[str] | None
55 | jetbrains_mode: bool
56 | languages: list[str]
57 | encoding: str | None
58 |
59 |
60 | class ResponseAvailableLanguages(BaseModel):
61 | languages: list[str]
62 |
63 |
64 | class RequestAddLanguage(BaseModel):
65 | language: str
66 |
67 |
68 | class RequestRemoveLanguage(BaseModel):
69 | language: str
70 |
71 |
72 | class RequestGetMemory(BaseModel):
73 | memory_name: str
74 |
75 |
76 | class ResponseGetMemory(BaseModel):
77 | content: str
78 | memory_name: str
79 |
80 |
81 | class RequestSaveMemory(BaseModel):
82 | memory_name: str
83 | content: str
84 |
85 |
86 | class RequestDeleteMemory(BaseModel):
87 | memory_name: str
88 |
89 |
90 | class ResponseGetSerenaConfig(BaseModel):
91 | content: str
92 |
93 |
94 | class RequestSaveSerenaConfig(BaseModel):
95 | content: str
96 |
97 |
98 | class RequestCancelTaskExecution(BaseModel):
99 | task_id: int
100 |
101 |
102 | class QueuedExecution(BaseModel):
103 | task_id: int
104 | is_running: bool
105 | name: str
106 | finished_successfully: bool
107 | logged: bool
108 |
109 | @classmethod
110 | def from_task_info(cls, task_info: TaskExecutor.TaskInfo) -> Self:
111 | return cls(
112 | task_id=task_info.task_id,
113 | is_running=task_info.is_running,
114 | name=task_info.name,
115 | finished_successfully=task_info.finished_successfully(),
116 | logged=task_info.logged,
117 | )
118 |
119 |
120 | class SerenaDashboardAPI:
121 | log = logging.getLogger(__qualname__)
122 |
123 | def __init__(
124 | self,
125 | memory_log_handler: MemoryLogHandler,
126 | tool_names: list[str],
127 | agent: "SerenaAgent",
128 | shutdown_callback: Callable[[], None] | None = None,
129 | tool_usage_stats: ToolUsageStats | None = None,
130 | ) -> None:
131 | self._memory_log_handler = memory_log_handler
132 | self._tool_names = tool_names
133 | self._agent = agent
134 | self._shutdown_callback = shutdown_callback
135 | self._app = Flask(__name__)
136 | self._tool_usage_stats = tool_usage_stats
137 | self._setup_routes()
138 |
139 | @property
140 | def memory_log_handler(self) -> MemoryLogHandler:
141 | return self._memory_log_handler
142 |
143 | def _setup_routes(self) -> None:
144 | # Static files
145 | @self._app.route("/dashboard/<path:filename>")
146 | def serve_dashboard(filename: str) -> Response:
147 | return send_from_directory(SERENA_DASHBOARD_DIR, filename)
148 |
149 | @self._app.route("/dashboard/")
150 | def serve_dashboard_index() -> Response:
151 | return send_from_directory(SERENA_DASHBOARD_DIR, "index.html")
152 |
153 | # API routes
154 |
155 | @self._app.route("/heartbeat", methods=["GET"])
156 | def get_heartbeat() -> dict[str, Any]:
157 | return {"status": "alive"}
158 |
159 | @self._app.route("/get_log_messages", methods=["POST"])
160 | def get_log_messages() -> dict[str, Any]:
161 | request_data = request.get_json()
162 | if not request_data:
163 | request_log = RequestLog()
164 | else:
165 | request_log = RequestLog.model_validate(request_data)
166 |
167 | result = self._get_log_messages(request_log)
168 | return result.model_dump()
169 |
170 | @self._app.route("/get_tool_names", methods=["GET"])
171 | def get_tool_names() -> dict[str, Any]:
172 | result = self._get_tool_names()
173 | return result.model_dump()
174 |
175 | @self._app.route("/get_tool_stats", methods=["GET"])
176 | def get_tool_stats_route() -> dict[str, Any]:
177 | result = self._get_tool_stats()
178 | return result.model_dump()
179 |
180 | @self._app.route("/clear_tool_stats", methods=["POST"])
181 | def clear_tool_stats_route() -> dict[str, str]:
182 | self._clear_tool_stats()
183 | return {"status": "cleared"}
184 |
185 | @self._app.route("/get_token_count_estimator_name", methods=["GET"])
186 | def get_token_count_estimator_name() -> dict[str, str]:
187 | estimator_name = self._tool_usage_stats.token_estimator_name if self._tool_usage_stats else "unknown"
188 | return {"token_count_estimator_name": estimator_name}
189 |
190 | @self._app.route("/get_config_overview", methods=["GET"])
191 | def get_config_overview() -> dict[str, Any]:
192 | result = self._agent.execute_task(self._get_config_overview, logged=False)
193 | return result.model_dump()
194 |
195 | @self._app.route("/shutdown", methods=["PUT"])
196 | def shutdown() -> dict[str, str]:
197 | self._shutdown()
198 | return {"status": "shutting down"}
199 |
200 | @self._app.route("/get_available_languages", methods=["GET"])
201 | def get_available_languages() -> dict[str, Any]:
202 | result = self._get_available_languages()
203 | return result.model_dump()
204 |
205 | @self._app.route("/add_language", methods=["POST"])
206 | def add_language() -> dict[str, str]:
207 | request_data = request.get_json()
208 | if not request_data:
209 | return {"status": "error", "message": "No data provided"}
210 | request_add_language = RequestAddLanguage.model_validate(request_data)
211 | try:
212 | self._add_language(request_add_language)
213 | return {"status": "success", "message": f"Language {request_add_language.language} added successfully"}
214 | except Exception as e:
215 | return {"status": "error", "message": str(e)}
216 |
217 | @self._app.route("/remove_language", methods=["POST"])
218 | def remove_language() -> dict[str, str]:
219 | request_data = request.get_json()
220 | if not request_data:
221 | return {"status": "error", "message": "No data provided"}
222 | request_remove_language = RequestRemoveLanguage.model_validate(request_data)
223 | try:
224 | self._remove_language(request_remove_language)
225 | return {"status": "success", "message": f"Language {request_remove_language.language} removed successfully"}
226 | except Exception as e:
227 | return {"status": "error", "message": str(e)}
228 |
229 | @self._app.route("/get_memory", methods=["POST"])
230 | def get_memory() -> dict[str, Any]:
231 | request_data = request.get_json()
232 | if not request_data:
233 | return {"status": "error", "message": "No data provided"}
234 | request_get_memory = RequestGetMemory.model_validate(request_data)
235 | try:
236 | result = self._get_memory(request_get_memory)
237 | return result.model_dump()
238 | except Exception as e:
239 | return {"status": "error", "message": str(e)}
240 |
241 | @self._app.route("/save_memory", methods=["POST"])
242 | def save_memory() -> dict[str, str]:
243 | request_data = request.get_json()
244 | if not request_data:
245 | return {"status": "error", "message": "No data provided"}
246 | request_save_memory = RequestSaveMemory.model_validate(request_data)
247 | try:
248 | self._save_memory(request_save_memory)
249 | return {"status": "success", "message": f"Memory {request_save_memory.memory_name} saved successfully"}
250 | except Exception as e:
251 | return {"status": "error", "message": str(e)}
252 |
253 | @self._app.route("/delete_memory", methods=["POST"])
254 | def delete_memory() -> dict[str, str]:
255 | request_data = request.get_json()
256 | if not request_data:
257 | return {"status": "error", "message": "No data provided"}
258 | request_delete_memory = RequestDeleteMemory.model_validate(request_data)
259 | try:
260 | self._delete_memory(request_delete_memory)
261 | return {"status": "success", "message": f"Memory {request_delete_memory.memory_name} deleted successfully"}
262 | except Exception as e:
263 | return {"status": "error", "message": str(e)}
264 |
265 | @self._app.route("/get_serena_config", methods=["GET"])
266 | def get_serena_config() -> dict[str, Any]:
267 | try:
268 | result = self._get_serena_config()
269 | return result.model_dump()
270 | except Exception as e:
271 | return {"status": "error", "message": str(e)}
272 |
273 | @self._app.route("/save_serena_config", methods=["POST"])
274 | def save_serena_config() -> dict[str, str]:
275 | request_data = request.get_json()
276 | if not request_data:
277 | return {"status": "error", "message": "No data provided"}
278 | request_save_config = RequestSaveSerenaConfig.model_validate(request_data)
279 | try:
280 | self._save_serena_config(request_save_config)
281 | return {"status": "success", "message": "Serena config saved successfully"}
282 | except Exception as e:
283 | return {"status": "error", "message": str(e)}
284 |
285 | @self._app.route("/queued_task_executions", methods=["GET"])
286 | def get_queued_executions() -> dict[str, Any]:
287 | try:
288 | current_executions = self._agent.get_current_tasks()
289 | response = [QueuedExecution.from_task_info(task_info).model_dump() for task_info in current_executions]
290 | return {"queued_executions": response, "status": "success"}
291 | except Exception as e:
292 | return {"status": "error", "message": str(e)}
293 |
294 | @self._app.route("/cancel_task_execution", methods=["POST"])
295 | def cancel_task_execution() -> dict[str, Any]:
296 | request_data = request.get_json()
297 | try:
298 | request_cancel_task = RequestCancelTaskExecution.model_validate(request_data)
299 | for task in self._agent.get_current_tasks():
300 | if task.task_id == request_cancel_task.task_id:
301 | task.cancel()
302 | return {"status": "success", "was_cancelled": True}
303 | return {
304 | "status": "success",
305 | "was_cancelled": False,
306 | "message": f"Task with id {request_data.get('task_id')} not found, maybe execution was already finished",
307 | }
308 | except Exception as e:
309 | return {"status": "error", "message": str(e), "was_cancelled": False}
310 |
311 | @self._app.route("/last_execution", methods=["GET"])
312 | def get_last_execution() -> dict[str, Any]:
313 | try:
314 | last_execution_info = self._agent.get_last_executed_task()
315 | response = QueuedExecution.from_task_info(last_execution_info).model_dump() if last_execution_info is not None else None
316 | return {"last_execution": response, "status": "success"}
317 | except Exception as e:
318 | return {"status": "error", "message": str(e)}
319 |
320 | def _get_log_messages(self, request_log: RequestLog) -> ResponseLog:
321 | all_messages = self._memory_log_handler.get_log_messages()
322 | requested_messages = all_messages[request_log.start_idx :] if request_log.start_idx <= len(all_messages) else []
323 | project = self._agent.get_active_project()
324 | project_name = project.project_name if project else None
325 | return ResponseLog(messages=requested_messages, max_idx=len(all_messages) - 1, active_project=project_name)
326 |
327 | def _get_tool_names(self) -> ResponseToolNames:
328 | return ResponseToolNames(tool_names=self._tool_names)
329 |
330 | def _get_tool_stats(self) -> ResponseToolStats:
331 | if self._tool_usage_stats is not None:
332 | return ResponseToolStats(stats=self._tool_usage_stats.get_tool_stats_dict())
333 | else:
334 | return ResponseToolStats(stats={})
335 |
336 | def _clear_tool_stats(self) -> None:
337 | if self._tool_usage_stats is not None:
338 | self._tool_usage_stats.clear()
339 |
340 | def _get_config_overview(self) -> ResponseConfigOverview:
341 | from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode
342 |
343 | # Get active project info
344 | project = self._agent.get_active_project()
345 | active_project_name = project.project_name if project else None
346 | project_info = {
347 | "name": active_project_name,
348 | "language": ", ".join([l.value for l in project.project_config.languages]) if project else None,
349 | "path": str(project.project_root) if project else None,
350 | }
351 |
352 | # Get context info
353 | context = self._agent.get_context()
354 | context_info = {
355 | "name": context.name,
356 | "description": context.description,
357 | "path": SerenaAgentContext.get_path(context.name, instance=context),
358 | }
359 |
360 | # Get active modes
361 | modes = self._agent.get_active_modes()
362 | modes_info = [
363 | {"name": mode.name, "description": mode.description, "path": SerenaAgentMode.get_path(mode.name, instance=mode)}
364 | for mode in modes
365 | ]
366 | active_mode_names = [mode.name for mode in modes]
367 |
368 | # Get active tools
369 | active_tools = self._agent.get_active_tool_names()
370 |
371 | # Get registered projects
372 | registered_projects: list[dict[str, str | bool]] = []
373 | for proj in self._agent.serena_config.projects:
374 | registered_projects.append(
375 | {
376 | "name": proj.project_name,
377 | "path": str(proj.project_root),
378 | "is_active": proj.project_name == active_project_name,
379 | }
380 | )
381 |
382 | # Get all available tools (excluding active ones)
383 | all_tool_names = sorted([tool.get_name_from_cls() for tool in self._agent._all_tools.values()])
384 | available_tools: list[dict[str, str | bool]] = []
385 | for tool_name in all_tool_names:
386 | if tool_name not in active_tools:
387 | available_tools.append(
388 | {
389 | "name": tool_name,
390 | "is_active": False,
391 | }
392 | )
393 |
394 | # Get all available modes
395 | all_mode_names = SerenaAgentMode.list_registered_mode_names()
396 | available_modes: list[dict[str, str | bool]] = []
397 | for mode_name in all_mode_names:
398 | try:
399 | mode_path = SerenaAgentMode.get_path(mode_name)
400 | except FileNotFoundError:
401 | # Skip modes that can't be found (shouldn't happen for registered modes)
402 | continue
403 | available_modes.append(
404 | {
405 | "name": mode_name,
406 | "is_active": mode_name in active_mode_names,
407 | "path": mode_path,
408 | }
409 | )
410 |
411 | # Get all available contexts
412 | all_context_names = SerenaAgentContext.list_registered_context_names()
413 | available_contexts: list[dict[str, str | bool]] = []
414 | for context_name in all_context_names:
415 | try:
416 | context_path = SerenaAgentContext.get_path(context_name)
417 | except FileNotFoundError:
418 | # Skip contexts that can't be found (shouldn't happen for registered contexts)
419 | continue
420 | available_contexts.append(
421 | {
422 | "name": context_name,
423 | "is_active": context_name == context.name,
424 | "path": context_path,
425 | }
426 | )
427 |
428 | # Get basic tool stats (just num_calls for overview)
429 | tool_stats_summary = {}
430 | if self._tool_usage_stats is not None:
431 | full_stats = self._tool_usage_stats.get_tool_stats_dict()
432 | tool_stats_summary = {name: {"num_calls": stats["num_times_called"]} for name, stats in full_stats.items()}
433 |
434 | # Get available memories if ReadMemoryTool is active
435 | available_memories = None
436 | if self._agent.tool_is_active("read_memory") and project is not None:
437 | available_memories = project.memories_manager.list_memories()
438 |
439 | # Get list of languages for the active project
440 | languages = []
441 | if project is not None:
442 | languages = [lang.value for lang in project.project_config.languages]
443 |
444 | # Get file encoding for the active project
445 | encoding = None
446 | if project is not None:
447 | encoding = project.project_config.encoding
448 |
449 | return ResponseConfigOverview(
450 | active_project=project_info,
451 | context=context_info,
452 | modes=modes_info,
453 | active_tools=active_tools,
454 | tool_stats_summary=tool_stats_summary,
455 | registered_projects=registered_projects,
456 | available_tools=available_tools,
457 | available_modes=available_modes,
458 | available_contexts=available_contexts,
459 | available_memories=available_memories,
460 | jetbrains_mode=self._agent.serena_config.language_backend == LanguageBackend.JETBRAINS,
461 | languages=languages,
462 | encoding=encoding,
463 | )
464 |
465 | def _shutdown(self) -> None:
466 | log.info("Shutting down Serena")
467 | if self._shutdown_callback:
468 | self._shutdown_callback()
469 | else:
470 | # noinspection PyProtectedMember
471 | # noinspection PyUnresolvedReferences
472 | os._exit(0)
473 |
474 | def _get_available_languages(self) -> ResponseAvailableLanguages:
475 | from solidlsp.ls_config import Language
476 |
477 | def run() -> ResponseAvailableLanguages:
478 | all_languages = [lang.value for lang in Language.iter_all(include_experimental=False)]
479 |
480 | # Filter out already added languages for the active project
481 | project = self._agent.get_active_project()
482 | if project:
483 | current_languages = [lang.value for lang in project.project_config.languages]
484 | available_languages = [lang for lang in all_languages if lang not in current_languages]
485 | else:
486 | available_languages = all_languages
487 |
488 | return ResponseAvailableLanguages(languages=sorted(available_languages))
489 |
490 | return self._agent.execute_task(run, logged=False)
491 |
492 | def _get_memory(self, request_get_memory: RequestGetMemory) -> ResponseGetMemory:
493 | def run() -> ResponseGetMemory:
494 | project = self._agent.get_active_project()
495 | if project is None:
496 | raise ValueError("No active project")
497 |
498 | content = project.memories_manager.load_memory(request_get_memory.memory_name)
499 | return ResponseGetMemory(content=content, memory_name=request_get_memory.memory_name)
500 |
501 | return self._agent.execute_task(run, logged=False)
502 |
503 | def _save_memory(self, request_save_memory: RequestSaveMemory) -> None:
504 | def run() -> None:
505 | project = self._agent.get_active_project()
506 | if project is None:
507 | raise ValueError("No active project")
508 |
509 | project.memories_manager.save_memory(request_save_memory.memory_name, request_save_memory.content)
510 |
511 | self._agent.execute_task(run, logged=True, name="SaveMemory")
512 |
513 | def _delete_memory(self, request_delete_memory: RequestDeleteMemory) -> None:
514 | def run() -> None:
515 | project = self._agent.get_active_project()
516 | if project is None:
517 | raise ValueError("No active project")
518 |
519 | project.memories_manager.delete_memory(request_delete_memory.memory_name)
520 |
521 | self._agent.execute_task(run, logged=True, name="DeleteMemory")
522 |
523 | def _get_serena_config(self) -> ResponseGetSerenaConfig:
524 | config_path = self._agent.serena_config.config_file_path
525 | if config_path is None or not os.path.exists(config_path):
526 | raise ValueError("Serena config file not found")
527 |
528 | with open(config_path, encoding="utf-8") as f:
529 | content = f.read()
530 |
531 | return ResponseGetSerenaConfig(content=content)
532 |
533 | def _save_serena_config(self, request_save_config: RequestSaveSerenaConfig) -> None:
534 | def run() -> None:
535 | config_path = self._agent.serena_config.config_file_path
536 | if config_path is None:
537 | raise ValueError("Serena config file path not set")
538 |
539 | with open(config_path, "w", encoding="utf-8") as f:
540 | f.write(request_save_config.content)
541 |
542 | self._agent.execute_task(run, logged=True, name="SaveSerenaConfig")
543 |
544 | def _add_language(self, request_add_language: RequestAddLanguage) -> None:
545 | from solidlsp.ls_config import Language
546 |
547 | try:
548 | language = Language(request_add_language.language)
549 | except ValueError:
550 | raise ValueError(f"Invalid language: {request_add_language.language}")
551 | # add_language is already thread-safe
552 | self._agent.add_language(language)
553 |
554 | def _remove_language(self, request_remove_language: RequestRemoveLanguage) -> None:
555 | from solidlsp.ls_config import Language
556 |
557 | try:
558 | language = Language(request_remove_language.language)
559 | except ValueError:
560 | raise ValueError(f"Invalid language: {request_remove_language.language}")
561 | # remove_language is already thread-safe
562 | self._agent.remove_language(language)
563 |
564 | @staticmethod
565 | def _find_first_free_port(start_port: int, host: str) -> int:
566 | port = start_port
567 | while port <= 65535:
568 | try:
569 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
570 | sock.bind((host, port))
571 | return port
572 | except OSError:
573 | port += 1
574 |
575 | raise RuntimeError(f"No free ports found starting from {start_port}")
576 |
577 | def run(self, host: str, port: int) -> int:
578 | """
579 | Runs the dashboard on the given host and port and returns the port number.
580 | """
581 | # patch flask.cli.show_server to avoid printing the server info
582 | from flask import cli
583 |
584 | cli.show_server_banner = lambda *args, **kwargs: None
585 |
586 | self._app.run(host=host, port=port, debug=False, use_reloader=False, threaded=True)
587 | return port
588 |
589 | def run_in_thread(self, host: str) -> tuple[threading.Thread, int]:
590 | port = self._find_first_free_port(0x5EDA, host)
591 | log.info("Starting dashboard (listen_address=%s, port=%d)", host, port)
592 | thread = threading.Thread(target=lambda: self.run(host=host, port=port), daemon=True)
593 | thread.start()
594 | return thread, port
595 |
```
--------------------------------------------------------------------------------
/src/solidlsp/language_servers/kotlin_language_server.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | Provides Kotlin specific instantiation of the LanguageServer class. Contains various configurations and settings specific to Kotlin.
3 |
4 | You can configure the following options in ls_specific_settings (in serena_config.yml):
5 |
6 | ls_specific_settings:
7 | kotlin:
8 | jvm_options: '-Xmx4G' # JVM options for Kotlin Language Server (default: -Xmx4G)
9 |
10 | Example configuration for large projects:
11 |
12 | ls_specific_settings:
13 | kotlin:
14 | jvm_options: '-Xmx8G -XX:+UseG1GC'
15 | """
16 |
17 | import dataclasses
18 | import logging
19 | import os
20 | import pathlib
21 | import stat
22 | from typing import cast
23 |
24 | from solidlsp.ls import SolidLanguageServer
25 | from solidlsp.ls_config import Language, LanguageServerConfig
26 | from solidlsp.ls_utils import FileUtils, PlatformUtils
27 | from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
28 | from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
29 | from solidlsp.settings import SolidLSPSettings
30 |
31 | log = logging.getLogger(__name__)
32 |
33 | # Default JVM options for Kotlin Language Server
34 | # -Xmx4G: Limit max heap to 4GB to prevent OOM on large projects
35 | DEFAULT_KOTLIN_JVM_OPTIONS = "-Xmx4G"
36 |
37 |
38 | @dataclasses.dataclass
39 | class KotlinRuntimeDependencyPaths:
40 | """
41 | Stores the paths to the runtime dependencies of Kotlin Language Server
42 | """
43 |
44 | java_path: str
45 | java_home_path: str
46 | kotlin_executable_path: str
47 |
48 |
49 | class KotlinLanguageServer(SolidLanguageServer):
50 | """
51 | Provides Kotlin specific instantiation of the LanguageServer class. Contains various configurations and settings specific to Kotlin.
52 | """
53 |
54 | def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
55 | """
56 | Creates a Kotlin Language Server instance. This class is not meant to be instantiated directly. Use LanguageServer.create() instead.
57 | """
58 | runtime_dependency_paths = self._setup_runtime_dependencies(config, solidlsp_settings)
59 | self.runtime_dependency_paths = runtime_dependency_paths
60 |
61 | # Create command to execute the Kotlin Language Server script
62 | cmd = [self.runtime_dependency_paths.kotlin_executable_path, "--stdio"]
63 |
64 | # Get JVM options from settings or use default
65 | jvm_options = DEFAULT_KOTLIN_JVM_OPTIONS
66 | if solidlsp_settings.ls_specific_settings:
67 | kotlin_settings = solidlsp_settings.get_ls_specific_settings(Language.KOTLIN)
68 | custom_jvm_options = kotlin_settings.get("jvm_options", "")
69 | if custom_jvm_options:
70 | jvm_options = custom_jvm_options
71 | log.info(f"Using custom JVM options for Kotlin Language Server: {jvm_options}")
72 |
73 | # Set environment variables including JAVA_HOME and JVM options
74 | # JAVA_TOOL_OPTIONS is automatically picked up by any Java process
75 | proc_env = {
76 | "JAVA_HOME": self.runtime_dependency_paths.java_home_path,
77 | "JAVA_TOOL_OPTIONS": jvm_options,
78 | }
79 |
80 | super().__init__(
81 | config, repository_root_path, ProcessLaunchInfo(cmd=cmd, env=proc_env, cwd=repository_root_path), "kotlin", solidlsp_settings
82 | )
83 |
84 | @classmethod
85 | def _setup_runtime_dependencies(cls, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings) -> KotlinRuntimeDependencyPaths:
86 | """
87 | Setup runtime dependencies for Kotlin Language Server and return the paths.
88 | """
89 | platform_id = PlatformUtils.get_platform_id()
90 |
91 | # Verify platform support
92 | assert (
93 | platform_id.value.startswith("win-") or platform_id.value.startswith("linux-") or platform_id.value.startswith("osx-")
94 | ), "Only Windows, Linux and macOS platforms are supported for Kotlin in multilspy at the moment"
95 |
96 | # Runtime dependency information
97 | runtime_dependencies = {
98 | "runtimeDependency": {
99 | "id": "KotlinLsp",
100 | "description": "Kotlin Language Server",
101 | "url": "https://download-cdn.jetbrains.com/kotlin-lsp/0.253.10629/kotlin-0.253.10629.zip",
102 | "archiveType": "zip",
103 | },
104 | "java": {
105 | "win-x64": {
106 | "url": "https://github.com/redhat-developer/vscode-java/releases/download/v1.42.0/java-win32-x64-1.42.0-561.vsix",
107 | "archiveType": "zip",
108 | "java_home_path": "extension/jre/21.0.7-win32-x86_64",
109 | "java_path": "extension/jre/21.0.7-win32-x86_64/bin/java.exe",
110 | },
111 | "linux-x64": {
112 | "url": "https://github.com/redhat-developer/vscode-java/releases/download/v1.42.0/java-linux-x64-1.42.0-561.vsix",
113 | "archiveType": "zip",
114 | "java_home_path": "extension/jre/21.0.7-linux-x86_64",
115 | "java_path": "extension/jre/21.0.7-linux-x86_64/bin/java",
116 | },
117 | "linux-arm64": {
118 | "url": "https://github.com/redhat-developer/vscode-java/releases/download/v1.42.0/java-linux-arm64-1.42.0-561.vsix",
119 | "archiveType": "zip",
120 | "java_home_path": "extension/jre/21.0.7-linux-aarch64",
121 | "java_path": "extension/jre/21.0.7-linux-aarch64/bin/java",
122 | },
123 | "osx-x64": {
124 | "url": "https://github.com/redhat-developer/vscode-java/releases/download/v1.42.0/java-darwin-x64-1.42.0-561.vsix",
125 | "archiveType": "zip",
126 | "java_home_path": "extension/jre/21.0.7-macosx-x86_64",
127 | "java_path": "extension/jre/21.0.7-macosx-x86_64/bin/java",
128 | },
129 | "osx-arm64": {
130 | "url": "https://github.com/redhat-developer/vscode-java/releases/download/v1.42.0/java-darwin-arm64-1.42.0-561.vsix",
131 | "archiveType": "zip",
132 | "java_home_path": "extension/jre/21.0.7-macosx-aarch64",
133 | "java_path": "extension/jre/21.0.7-macosx-aarch64/bin/java",
134 | },
135 | },
136 | }
137 |
138 | kotlin_dependency = runtime_dependencies["runtimeDependency"]
139 | java_dependency = runtime_dependencies["java"][platform_id.value] # type: ignore
140 |
141 | # Setup paths for dependencies
142 | static_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "kotlin_language_server")
143 | os.makedirs(static_dir, exist_ok=True)
144 |
145 | # Setup Java paths
146 | java_dir = os.path.join(static_dir, "java")
147 | os.makedirs(java_dir, exist_ok=True)
148 |
149 | java_home_path = os.path.join(java_dir, java_dependency["java_home_path"])
150 | java_path = os.path.join(java_dir, java_dependency["java_path"])
151 |
152 | # Download and extract Java if not exists
153 | if not os.path.exists(java_path):
154 | log.info(f"Downloading Java for {platform_id.value}...")
155 | FileUtils.download_and_extract_archive(java_dependency["url"], java_dir, java_dependency["archiveType"])
156 | # Make Java executable
157 | if not platform_id.value.startswith("win-"):
158 | os.chmod(java_path, 0o755)
159 |
160 | assert os.path.exists(java_path), f"Java executable not found at {java_path}"
161 |
162 | # Setup Kotlin Language Server paths
163 | kotlin_ls_dir = static_dir
164 |
165 | # Get platform-specific executable script path
166 | if platform_id.value.startswith("win-"):
167 | kotlin_script = os.path.join(kotlin_ls_dir, "kotlin-lsp.cmd")
168 | else:
169 | kotlin_script = os.path.join(kotlin_ls_dir, "kotlin-lsp.sh")
170 |
171 | # Download and extract Kotlin Language Server if script doesn't exist
172 | if not os.path.exists(kotlin_script):
173 | log.info("Downloading Kotlin Language Server...")
174 | FileUtils.download_and_extract_archive(kotlin_dependency["url"], static_dir, kotlin_dependency["archiveType"]) # type: ignore
175 |
176 | # Make script executable on Unix platforms
177 | if os.path.exists(kotlin_script) and not platform_id.value.startswith("win-"):
178 | os.chmod(
179 | kotlin_script, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
180 | )
181 |
182 | # Use script file
183 | if os.path.exists(kotlin_script):
184 | kotlin_executable_path = kotlin_script
185 | log.info(f"Using Kotlin Language Server script at {kotlin_script}")
186 | else:
187 | raise FileNotFoundError(f"Kotlin Language Server script not found at {kotlin_script}")
188 |
189 | return KotlinRuntimeDependencyPaths(
190 | java_path=java_path, java_home_path=java_home_path, kotlin_executable_path=kotlin_executable_path
191 | )
192 |
193 | @staticmethod
194 | def _get_initialize_params(repository_absolute_path: str) -> InitializeParams:
195 | """
196 | Returns the initialize params for the Kotlin Language Server.
197 | """
198 | if not os.path.isabs(repository_absolute_path):
199 | repository_absolute_path = os.path.abspath(repository_absolute_path)
200 |
201 | root_uri = pathlib.Path(repository_absolute_path).as_uri()
202 | initialize_params = {
203 | "clientInfo": {"name": "Multilspy Kotlin Client", "version": "1.0.0"},
204 | "locale": "en",
205 | "rootPath": repository_absolute_path,
206 | "rootUri": root_uri,
207 | "capabilities": {
208 | "workspace": {
209 | "applyEdit": True,
210 | "workspaceEdit": {
211 | "documentChanges": True,
212 | "resourceOperations": ["create", "rename", "delete"],
213 | "failureHandling": "textOnlyTransactional",
214 | "normalizesLineEndings": True,
215 | "changeAnnotationSupport": {"groupsOnLabel": True},
216 | },
217 | "didChangeConfiguration": {"dynamicRegistration": True},
218 | "didChangeWatchedFiles": {"dynamicRegistration": True, "relativePatternSupport": True},
219 | "symbol": {
220 | "dynamicRegistration": True,
221 | "symbolKind": {"valueSet": list(range(1, 27))},
222 | "tagSupport": {"valueSet": [1]},
223 | "resolveSupport": {"properties": ["location.range"]},
224 | },
225 | "codeLens": {"refreshSupport": True},
226 | "executeCommand": {"dynamicRegistration": True},
227 | "configuration": True,
228 | "workspaceFolders": True,
229 | "semanticTokens": {"refreshSupport": True},
230 | "fileOperations": {
231 | "dynamicRegistration": True,
232 | "didCreate": True,
233 | "didRename": True,
234 | "didDelete": True,
235 | "willCreate": True,
236 | "willRename": True,
237 | "willDelete": True,
238 | },
239 | "inlineValue": {"refreshSupport": True},
240 | "inlayHint": {"refreshSupport": True},
241 | "diagnostics": {"refreshSupport": True},
242 | },
243 | "textDocument": {
244 | "publishDiagnostics": {
245 | "relatedInformation": True,
246 | "versionSupport": False,
247 | "tagSupport": {"valueSet": [1, 2]},
248 | "codeDescriptionSupport": True,
249 | "dataSupport": True,
250 | },
251 | "synchronization": {"dynamicRegistration": True, "willSave": True, "willSaveWaitUntil": True, "didSave": True},
252 | "completion": {
253 | "dynamicRegistration": True,
254 | "contextSupport": True,
255 | "completionItem": {
256 | "snippetSupport": False,
257 | "commitCharactersSupport": True,
258 | "documentationFormat": ["markdown", "plaintext"],
259 | "deprecatedSupport": True,
260 | "preselectSupport": True,
261 | "tagSupport": {"valueSet": [1]},
262 | "insertReplaceSupport": False,
263 | "resolveSupport": {"properties": ["documentation", "detail", "additionalTextEdits"]},
264 | "insertTextModeSupport": {"valueSet": [1, 2]},
265 | "labelDetailsSupport": True,
266 | },
267 | "insertTextMode": 2,
268 | "completionItemKind": {
269 | "valueSet": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]
270 | },
271 | "completionList": {"itemDefaults": ["commitCharacters", "editRange", "insertTextFormat", "insertTextMode"]},
272 | },
273 | "hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]},
274 | "signatureHelp": {
275 | "dynamicRegistration": True,
276 | "signatureInformation": {
277 | "documentationFormat": ["markdown", "plaintext"],
278 | "parameterInformation": {"labelOffsetSupport": True},
279 | "activeParameterSupport": True,
280 | },
281 | "contextSupport": True,
282 | },
283 | "definition": {"dynamicRegistration": True, "linkSupport": True},
284 | "references": {"dynamicRegistration": True},
285 | "documentHighlight": {"dynamicRegistration": True},
286 | "documentSymbol": {
287 | "dynamicRegistration": True,
288 | "symbolKind": {"valueSet": list(range(1, 27))},
289 | "hierarchicalDocumentSymbolSupport": True,
290 | "tagSupport": {"valueSet": [1]},
291 | "labelSupport": True,
292 | },
293 | "codeAction": {
294 | "dynamicRegistration": True,
295 | "isPreferredSupport": True,
296 | "disabledSupport": True,
297 | "dataSupport": True,
298 | "resolveSupport": {"properties": ["edit"]},
299 | "codeActionLiteralSupport": {
300 | "codeActionKind": {
301 | "valueSet": [
302 | "",
303 | "quickfix",
304 | "refactor",
305 | "refactor.extract",
306 | "refactor.inline",
307 | "refactor.rewrite",
308 | "source",
309 | "source.organizeImports",
310 | ]
311 | }
312 | },
313 | "honorsChangeAnnotations": False,
314 | },
315 | "codeLens": {"dynamicRegistration": True},
316 | "formatting": {"dynamicRegistration": True},
317 | "rangeFormatting": {"dynamicRegistration": True},
318 | "onTypeFormatting": {"dynamicRegistration": True},
319 | "rename": {
320 | "dynamicRegistration": True,
321 | "prepareSupport": True,
322 | "prepareSupportDefaultBehavior": 1,
323 | "honorsChangeAnnotations": True,
324 | },
325 | "documentLink": {"dynamicRegistration": True, "tooltipSupport": True},
326 | "typeDefinition": {"dynamicRegistration": True, "linkSupport": True},
327 | "implementation": {"dynamicRegistration": True, "linkSupport": True},
328 | "colorProvider": {"dynamicRegistration": True},
329 | "foldingRange": {
330 | "dynamicRegistration": True,
331 | "rangeLimit": 5000,
332 | "lineFoldingOnly": True,
333 | "foldingRangeKind": {"valueSet": ["comment", "imports", "region"]},
334 | "foldingRange": {"collapsedText": False},
335 | },
336 | "declaration": {"dynamicRegistration": True, "linkSupport": True},
337 | "selectionRange": {"dynamicRegistration": True},
338 | "callHierarchy": {"dynamicRegistration": True},
339 | "semanticTokens": {
340 | "dynamicRegistration": True,
341 | "tokenTypes": [
342 | "namespace",
343 | "type",
344 | "class",
345 | "enum",
346 | "interface",
347 | "struct",
348 | "typeParameter",
349 | "parameter",
350 | "variable",
351 | "property",
352 | "enumMember",
353 | "event",
354 | "function",
355 | "method",
356 | "macro",
357 | "keyword",
358 | "modifier",
359 | "comment",
360 | "string",
361 | "number",
362 | "regexp",
363 | "operator",
364 | "decorator",
365 | ],
366 | "tokenModifiers": [
367 | "declaration",
368 | "definition",
369 | "readonly",
370 | "static",
371 | "deprecated",
372 | "abstract",
373 | "async",
374 | "modification",
375 | "documentation",
376 | "defaultLibrary",
377 | ],
378 | "formats": ["relative"],
379 | "requests": {"range": True, "full": {"delta": True}},
380 | "multilineTokenSupport": False,
381 | "overlappingTokenSupport": False,
382 | "serverCancelSupport": True,
383 | "augmentsSyntaxTokens": True,
384 | },
385 | "linkedEditingRange": {"dynamicRegistration": True},
386 | "typeHierarchy": {"dynamicRegistration": True},
387 | "inlineValue": {"dynamicRegistration": True},
388 | "inlayHint": {
389 | "dynamicRegistration": True,
390 | "resolveSupport": {"properties": ["tooltip", "textEdits", "label.tooltip", "label.location", "label.command"]},
391 | },
392 | "diagnostic": {"dynamicRegistration": True, "relatedDocumentSupport": False},
393 | },
394 | "window": {
395 | "showMessage": {"messageActionItem": {"additionalPropertiesSupport": True}},
396 | "showDocument": {"support": True},
397 | "workDoneProgress": True,
398 | },
399 | "general": {
400 | "staleRequestSupport": {
401 | "cancel": True,
402 | "retryOnContentModified": [
403 | "textDocument/semanticTokens/full",
404 | "textDocument/semanticTokens/range",
405 | "textDocument/semanticTokens/full/delta",
406 | ],
407 | },
408 | "regularExpressions": {"engine": "ECMAScript", "version": "ES2020"},
409 | "markdown": {"parser": "marked", "version": "1.1.0"},
410 | "positionEncodings": ["utf-16"],
411 | },
412 | "notebookDocument": {"synchronization": {"dynamicRegistration": True, "executionSummarySupport": True}},
413 | },
414 | "initializationOptions": {
415 | "workspaceFolders": [root_uri],
416 | "storagePath": None,
417 | "codegen": {"enabled": False},
418 | "compiler": {"jvm": {"target": "default"}},
419 | "completion": {"snippets": {"enabled": True}},
420 | "diagnostics": {"enabled": True, "level": 4, "debounceTime": 250},
421 | "scripts": {"enabled": True, "buildScriptsEnabled": True},
422 | "indexing": {"enabled": True},
423 | "externalSources": {"useKlsScheme": False, "autoConvertToKotlin": False},
424 | "inlayHints": {"typeHints": False, "parameterHints": False, "chainedHints": False},
425 | "formatting": {
426 | "formatter": "ktfmt",
427 | "ktfmt": {
428 | "style": "google",
429 | "indent": 4,
430 | "maxWidth": 100,
431 | "continuationIndent": 8,
432 | "removeUnusedImports": True,
433 | },
434 | },
435 | },
436 | "trace": "verbose",
437 | "processId": os.getpid(),
438 | "workspaceFolders": [
439 | {
440 | "uri": root_uri,
441 | "name": os.path.basename(repository_absolute_path),
442 | }
443 | ],
444 | }
445 | return cast(InitializeParams, initialize_params)
446 |
447 | def _start_server(self) -> None:
448 | """
449 | Starts the Kotlin Language Server
450 | """
451 |
452 | def execute_client_command_handler(params: dict) -> list:
453 | return []
454 |
455 | def do_nothing(params: dict) -> None:
456 | return
457 |
458 | def window_log_message(msg: dict) -> None:
459 | log.info(f"LSP: window/logMessage: {msg}")
460 |
461 | self.server.on_request("client/registerCapability", do_nothing)
462 | self.server.on_notification("language/status", do_nothing)
463 | self.server.on_notification("window/logMessage", window_log_message)
464 | self.server.on_request("workspace/executeClientCommand", execute_client_command_handler)
465 | self.server.on_notification("$/progress", do_nothing)
466 | self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
467 | self.server.on_notification("language/actionableNotification", do_nothing)
468 |
469 | log.info("Starting Kotlin server process")
470 | self.server.start()
471 | initialize_params = self._get_initialize_params(self.repository_root_path)
472 |
473 | log.info("Sending initialize request from LSP client to LSP server and awaiting response")
474 | init_response = self.server.send.initialize(initialize_params)
475 |
476 | capabilities = init_response["capabilities"]
477 | assert "textDocumentSync" in capabilities, "Server must support textDocumentSync"
478 | assert "hoverProvider" in capabilities, "Server must support hover"
479 | assert "completionProvider" in capabilities, "Server must support code completion"
480 | assert "signatureHelpProvider" in capabilities, "Server must support signature help"
481 | assert "definitionProvider" in capabilities, "Server must support go to definition"
482 | assert "referencesProvider" in capabilities, "Server must support find references"
483 | assert "documentSymbolProvider" in capabilities, "Server must support document symbols"
484 | assert "workspaceSymbolProvider" in capabilities, "Server must support workspace symbols"
485 | assert "semanticTokensProvider" in capabilities, "Server must support semantic tokens"
486 |
487 | self.server.notify.initialized({})
488 | self.completions_available.set()
489 |
```
--------------------------------------------------------------------------------
/src/solidlsp/language_servers/matlab_language_server.py:
--------------------------------------------------------------------------------
```python
1 | """
2 | MATLAB language server integration using the official MathWorks MATLAB Language Server.
3 |
4 | Architecture:
5 | This module uses the MathWorks MATLAB VS Code extension (mathworks.language-matlab)
6 | which contains a Node.js-based language server. The extension is downloaded from the
7 | VS Code Marketplace and extracted locally. The language server spawns a real MATLAB
8 | process to provide code intelligence - it is NOT a standalone static analyzer.
9 |
10 | Flow: Serena -> Node.js LSP Server -> MATLAB Process -> Code Analysis
11 |
12 | Why MATLAB installation is required:
13 | The language server launches an actual MATLAB session (via MatlabSession.js) to perform
14 | code analysis, diagnostics, and other features. Without MATLAB, the LSP cannot function.
15 | This is different from purely static analyzers that parse code without execution.
16 |
17 | Requirements:
18 | - MATLAB R2021b or later must be installed and licensed
19 | - Node.js must be installed (for running the language server)
20 | - MATLAB path can be specified via MATLAB_PATH environment variable or auto-detected
21 |
22 | The MATLAB language server provides:
23 | - Code diagnostics (publishDiagnostics)
24 | - Code completions (completionProvider)
25 | - Go to definition (definitionProvider)
26 | - Find references (referencesProvider)
27 | - Document symbols (documentSymbol)
28 | - Document formatting (documentFormattingProvider)
29 | - Function signature help (signatureHelpProvider)
30 | - Symbol rename (renameProvider)
31 | """
32 |
33 | import glob
34 | import logging
35 | import os
36 | import pathlib
37 | import platform
38 | import shutil
39 | import threading
40 | import zipfile
41 | from typing import Any, cast
42 |
43 | import requests
44 |
45 | from solidlsp.ls import LSPFileBuffer, SolidLanguageServer
46 | from solidlsp.ls_config import LanguageServerConfig
47 | from solidlsp.lsp_protocol_handler.lsp_types import DocumentSymbol, InitializeParams, SymbolInformation
48 | from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
49 | from solidlsp.settings import SolidLSPSettings
50 |
51 | log = logging.getLogger(__name__)
52 |
53 | # Environment variable for MATLAB installation path
54 | MATLAB_PATH_ENV_VAR = "MATLAB_PATH"
55 |
56 | # VS Code Marketplace URL for MATLAB extension
57 | MATLAB_EXTENSION_URL = (
58 | "https://marketplace.visualstudio.com/_apis/public/gallery/publishers/MathWorks/vsextensions/language-matlab/latest/vspackage"
59 | )
60 |
61 |
62 | class MatlabLanguageServer(SolidLanguageServer):
63 | """
64 | Provides MATLAB specific instantiation of the LanguageServer class using the official
65 | MathWorks MATLAB Language Server.
66 |
67 | The MATLAB language server requires:
68 | - MATLAB R2021b or later installed on the system
69 | - Node.js for running the language server
70 |
71 | The language server is automatically downloaded from the VS Code marketplace
72 | (MathWorks.language-matlab extension) and extracted.
73 |
74 | You can pass the following entries in ls_specific_settings["matlab"]:
75 | - matlab_path: Path to MATLAB installation (overrides MATLAB_PATH env var)
76 | """
77 |
78 | @staticmethod
79 | def _find_matlab_installation() -> str:
80 | """
81 | Find MATLAB installation path.
82 |
83 | Search order:
84 | 1. MATLAB_PATH environment variable
85 | 2. Common installation locations based on platform
86 |
87 | Returns:
88 | Path to MATLAB installation directory.
89 |
90 | Raises:
91 | RuntimeError: If MATLAB installation is not found.
92 |
93 | """
94 | # Check environment variable first
95 | matlab_path = os.environ.get(MATLAB_PATH_ENV_VAR)
96 | if matlab_path and os.path.isdir(matlab_path):
97 | log.info(f"Using MATLAB from environment variable {MATLAB_PATH_ENV_VAR}: {matlab_path}")
98 | return matlab_path
99 |
100 | system = platform.system()
101 |
102 | if system == "Darwin": # macOS
103 | # Check common macOS locations
104 | search_patterns = [
105 | "/Applications/MATLAB_*.app",
106 | "/Volumes/*/Applications/MATLAB_*.app",
107 | os.path.expanduser("~/Applications/MATLAB_*.app"),
108 | ]
109 | for pattern in search_patterns:
110 | matches = sorted(glob.glob(pattern), reverse=True) # Newest version first
111 | for match in matches:
112 | if os.path.isdir(match):
113 | log.info(f"Found MATLAB installation: {match}")
114 | return match
115 |
116 | elif system == "Windows":
117 | # Check common Windows locations
118 | search_patterns = [
119 | "C:\\Program Files\\MATLAB\\R*",
120 | "C:\\Program Files (x86)\\MATLAB\\R*",
121 | ]
122 | for pattern in search_patterns:
123 | matches = sorted(glob.glob(pattern), reverse=True)
124 | for match in matches:
125 | if os.path.isdir(match):
126 | log.info(f"Found MATLAB installation: {match}")
127 | return match
128 |
129 | elif system == "Linux":
130 | # Check common Linux locations
131 | search_patterns = [
132 | "/usr/local/MATLAB/R*",
133 | "/opt/MATLAB/R*",
134 | os.path.expanduser("~/MATLAB/R*"),
135 | ]
136 | for pattern in search_patterns:
137 | matches = sorted(glob.glob(pattern), reverse=True)
138 | for match in matches:
139 | if os.path.isdir(match):
140 | log.info(f"Found MATLAB installation: {match}")
141 | return match
142 |
143 | raise RuntimeError(
144 | f"MATLAB installation not found. Set the {MATLAB_PATH_ENV_VAR} environment variable "
145 | "to your MATLAB installation directory (e.g., /Applications/MATLAB_R2024b.app on macOS, "
146 | "C:\\Program Files\\MATLAB\\R2024b on Windows, or /usr/local/MATLAB/R2024b on Linux)."
147 | )
148 |
149 | def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
150 | """
151 | Creates a MatlabLanguageServer instance. This class is not meant to be instantiated directly.
152 | Use LanguageServer.create() instead.
153 | """
154 | matlab_lsp_command, matlab_path = self._setup_runtime_dependencies(config, solidlsp_settings)
155 | self._matlab_path = matlab_path
156 |
157 | # Set environment for MATLAB
158 | proc_env = {
159 | "MATLAB_INSTALL_PATH": matlab_path,
160 | }
161 |
162 | super().__init__(
163 | config,
164 | repository_root_path,
165 | ProcessLaunchInfo(cmd=matlab_lsp_command, cwd=repository_root_path, env=proc_env),
166 | "matlab",
167 | solidlsp_settings,
168 | )
169 | self.server_ready = threading.Event()
170 | self.initialize_searcher_command_available = threading.Event()
171 |
172 | @classmethod
173 | def _download_matlab_extension(cls, url: str, target_dir: str) -> bool:
174 | """
175 | Download and extract the MATLAB extension from VS Code marketplace.
176 |
177 | The VS Code marketplace packages extensions as .vsix files (which are ZIP archives).
178 | This method downloads the VSIX file and extracts it to get the language server.
179 |
180 | Args:
181 | url: VS Code marketplace URL for the MATLAB extension
182 | target_dir: Directory where the extension will be extracted
183 |
184 | Returns:
185 | True if successful, False otherwise
186 |
187 | """
188 | try:
189 | log.info(f"Downloading MATLAB extension from {url}")
190 |
191 | # Create target directory for the extension
192 | os.makedirs(target_dir, exist_ok=True)
193 |
194 | # Download with proper headers to mimic VS Code marketplace client
195 | headers = {
196 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
197 | "Accept": "application/octet-stream, application/vsix, */*",
198 | }
199 |
200 | response = requests.get(url, headers=headers, stream=True, timeout=300)
201 | response.raise_for_status()
202 |
203 | # Save to temporary VSIX file
204 | temp_file = os.path.join(target_dir, "matlab_extension_temp.vsix")
205 | total_size = int(response.headers.get("content-length", 0))
206 |
207 | log.info(f"Downloading {total_size / 1024 / 1024:.1f} MB...")
208 |
209 | with open(temp_file, "wb") as f:
210 | downloaded = 0
211 | for chunk in response.iter_content(chunk_size=8192):
212 | if chunk:
213 | f.write(chunk)
214 | downloaded += len(chunk)
215 | if total_size > 0 and downloaded % (10 * 1024 * 1024) == 0:
216 | progress = (downloaded / total_size) * 100
217 | log.info(f"Download progress: {progress:.1f}%")
218 |
219 | log.info("Download complete, extracting...")
220 |
221 | # Extract VSIX file (VSIX files are ZIP archives)
222 | with zipfile.ZipFile(temp_file, "r") as zip_ref:
223 | zip_ref.extractall(target_dir)
224 |
225 | # Clean up temp file
226 | os.remove(temp_file)
227 |
228 | log.info("MATLAB extension extracted successfully")
229 | return True
230 |
231 | except Exception as e:
232 | log.error(f"Error downloading/extracting MATLAB extension: {e}")
233 | return False
234 |
235 | @classmethod
236 | def _find_matlab_extension(cls, solidlsp_settings: SolidLSPSettings) -> str | None:
237 | """
238 | Find MATLAB extension in various locations.
239 |
240 | Search order:
241 | 1. Environment variable (MATLAB_EXTENSION_PATH)
242 | 2. Default download location (~/.serena/ls_resources/matlab-extension)
243 | 3. VS Code installed extensions
244 |
245 | Returns:
246 | Path to MATLAB extension directory or None if not found
247 |
248 | """
249 | # Check environment variable
250 | env_path = os.environ.get("MATLAB_EXTENSION_PATH")
251 | if env_path and os.path.exists(env_path):
252 | log.debug(f"Found MATLAB extension via MATLAB_EXTENSION_PATH: {env_path}")
253 | return env_path
254 | elif env_path:
255 | log.warning(f"MATLAB_EXTENSION_PATH set but directory not found: {env_path}")
256 |
257 | # Check default download location
258 | default_path = os.path.join(cls.ls_resources_dir(solidlsp_settings), "matlab-extension", "extension")
259 | if os.path.exists(default_path):
260 | log.debug(f"Found MATLAB extension in default location: {default_path}")
261 | return default_path
262 |
263 | # Search VS Code extensions
264 | vscode_extensions_dir = os.path.expanduser("~/.vscode/extensions")
265 | if os.path.exists(vscode_extensions_dir):
266 | for entry in os.listdir(vscode_extensions_dir):
267 | if entry.startswith("mathworks.language-matlab"):
268 | ext_path = os.path.join(vscode_extensions_dir, entry)
269 | if os.path.isdir(ext_path):
270 | log.debug(f"Found MATLAB extension in VS Code: {ext_path}")
271 | return ext_path
272 |
273 | log.debug("MATLAB extension not found in any known location")
274 | return None
275 |
276 | @classmethod
277 | def _download_and_install_matlab_extension(cls, solidlsp_settings: SolidLSPSettings) -> str | None:
278 | """
279 | Download and install MATLAB extension from VS Code marketplace.
280 |
281 | Returns:
282 | Path to installed extension or None if download failed
283 |
284 | """
285 | matlab_extension_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "matlab-extension")
286 |
287 | log.info(f"Downloading MATLAB extension from: {MATLAB_EXTENSION_URL}")
288 |
289 | if cls._download_matlab_extension(MATLAB_EXTENSION_URL, matlab_extension_dir):
290 | extension_path = os.path.join(matlab_extension_dir, "extension")
291 | if os.path.exists(extension_path):
292 | log.info("MATLAB extension downloaded and installed successfully")
293 | return extension_path
294 | else:
295 | log.error(f"Download completed but extension not found at: {extension_path}")
296 | else:
297 | log.error("Failed to download MATLAB extension from marketplace")
298 |
299 | return None
300 |
301 | @classmethod
302 | def _get_executable_path(cls, extension_path: str, system: str) -> str:
303 | """
304 | Get the path to the MATLAB language server executable based on platform.
305 |
306 | The language server is a Node.js script located in the extension's server directory.
307 | """
308 | # The MATLAB extension bundles the language server in the 'server' directory
309 | server_dir = os.path.join(extension_path, "server", "out")
310 | main_script = os.path.join(server_dir, "index.js")
311 |
312 | if os.path.exists(main_script):
313 | return main_script
314 |
315 | # Alternative location
316 | alt_script = os.path.join(extension_path, "out", "index.js")
317 | if os.path.exists(alt_script):
318 | return alt_script
319 |
320 | raise RuntimeError(f"MATLAB language server script not found in extension at {extension_path}")
321 |
322 | @classmethod
323 | def _setup_runtime_dependencies(cls, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings) -> tuple[list[str], str]:
324 | """
325 | Setup runtime dependencies for MATLAB Language Server and return the command to start the server.
326 |
327 | Returns:
328 | Tuple of (command to start the server, MATLAB installation path)
329 |
330 | """
331 | system = platform.system()
332 |
333 | # Verify node is installed
334 | node_path = shutil.which("node")
335 | if node_path is None:
336 | raise RuntimeError("Node.js is not installed or isn't in PATH. Please install Node.js and try again.")
337 |
338 | # Get MATLAB path from settings or auto-detect
339 | language_specific_config = solidlsp_settings.get_ls_specific_settings(cls.get_language_enum_instance())
340 | matlab_path = language_specific_config.get("matlab_path")
341 |
342 | if not matlab_path:
343 | matlab_path = cls._find_matlab_installation() # Raises RuntimeError if not found
344 |
345 | # Verify MATLAB path exists
346 | if not os.path.isdir(matlab_path):
347 | raise RuntimeError(f"MATLAB installation directory does not exist: {matlab_path}")
348 |
349 | log.info(f"Using MATLAB installation: {matlab_path}")
350 |
351 | # Find existing extension or download if needed
352 | extension_path = cls._find_matlab_extension(solidlsp_settings)
353 | if extension_path is None:
354 | log.info("MATLAB extension not found on disk, attempting to download...")
355 | extension_path = cls._download_and_install_matlab_extension(solidlsp_settings)
356 |
357 | if extension_path is None:
358 | raise RuntimeError(
359 | "Failed to locate or download MATLAB Language Server. Please either:\n"
360 | "1. Set MATLAB_EXTENSION_PATH environment variable to the MATLAB extension directory\n"
361 | "2. Install the MATLAB extension in VS Code (MathWorks.language-matlab)\n"
362 | "3. Ensure internet connection for automatic download"
363 | )
364 |
365 | # Get the language server script path
366 | server_script = cls._get_executable_path(extension_path, system)
367 |
368 | if not os.path.exists(server_script):
369 | raise RuntimeError(f"MATLAB Language Server script not found at: {server_script}")
370 |
371 | # Build the command to run the language server
372 | # The MATLAB language server is run via Node.js with the --stdio flag
373 | cmd = [node_path, server_script, "--stdio"]
374 |
375 | return cmd, matlab_path
376 |
377 | @staticmethod
378 | def _get_initialize_params(repository_absolute_path: str) -> InitializeParams:
379 | """Return the initialize params for the MATLAB Language Server."""
380 | root_uri = pathlib.Path(repository_absolute_path).as_uri()
381 | initialize_params = {
382 | "locale": "en",
383 | "capabilities": {
384 | "textDocument": {
385 | "synchronization": {"didSave": True, "dynamicRegistration": True},
386 | "completion": {
387 | "dynamicRegistration": True,
388 | "completionItem": {"snippetSupport": True},
389 | },
390 | "definition": {"dynamicRegistration": True},
391 | "references": {"dynamicRegistration": True},
392 | "documentSymbol": {
393 | "dynamicRegistration": True,
394 | "hierarchicalDocumentSymbolSupport": True,
395 | "symbolKind": {"valueSet": list(range(1, 27))},
396 | },
397 | "hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]},
398 | "signatureHelp": {"dynamicRegistration": True},
399 | "codeAction": {"dynamicRegistration": True},
400 | "formatting": {"dynamicRegistration": True},
401 | "rename": {"dynamicRegistration": True, "prepareSupport": True},
402 | "publishDiagnostics": {"relatedInformation": True},
403 | },
404 | "workspace": {
405 | "workspaceFolders": True,
406 | "didChangeConfiguration": {"dynamicRegistration": True},
407 | "symbol": {"dynamicRegistration": True},
408 | },
409 | },
410 | "processId": os.getpid(),
411 | "rootPath": repository_absolute_path,
412 | "rootUri": root_uri,
413 | "workspaceFolders": [
414 | {
415 | "uri": root_uri,
416 | "name": os.path.basename(repository_absolute_path),
417 | }
418 | ],
419 | }
420 | return cast(InitializeParams, initialize_params)
421 |
422 | def _start_server(self) -> None:
423 | """Start the MATLAB Language Server and wait for it to be ready."""
424 | root_uri = pathlib.Path(self.repository_root_path).as_uri()
425 |
426 | def register_capability_handler(params: dict) -> None:
427 | assert "registrations" in params
428 | for registration in params["registrations"]:
429 | if registration["method"] == "workspace/executeCommand":
430 | self.initialize_searcher_command_available.set()
431 | return
432 |
433 | def execute_client_command_handler(params: dict) -> list:
434 | return []
435 |
436 | def workspace_folders_handler(params: dict) -> list:
437 | """Handle workspace/workspaceFolders request from the server."""
438 | return [{"uri": root_uri, "name": os.path.basename(self.repository_root_path)}]
439 |
440 | def workspace_configuration_handler(params: dict) -> list:
441 | """Handle workspace/configuration request from the server."""
442 | items = params.get("items", [])
443 | result = []
444 | for item in items:
445 | section = item.get("section", "")
446 | if section == "MATLAB":
447 | # Return MATLAB configuration
448 | result.append({"installPath": self._matlab_path, "matlabConnectionTiming": "onStart"})
449 | else:
450 | result.append({})
451 | return result
452 |
453 | def do_nothing(params: dict) -> None:
454 | return
455 |
456 | def window_log_message(msg: dict) -> None:
457 | log.info(f"LSP: window/logMessage: {msg}")
458 | message_text = msg.get("message", "")
459 | # Check for MATLAB language server ready signals
460 | # Wait for "MVM attach success" or "Adding workspace folder" which indicates MATLAB is fully ready
461 | # Note: "connected to" comes earlier but the server isn't fully ready at that point
462 | if "mvm attach success" in message_text.lower() or "adding workspace folder" in message_text.lower():
463 | log.info("MATLAB language server ready signal detected (MVM attached)")
464 | self.server_ready.set()
465 | self.completions_available.set()
466 |
467 | self.server.on_request("client/registerCapability", register_capability_handler)
468 | self.server.on_notification("window/logMessage", window_log_message)
469 | self.server.on_request("workspace/executeClientCommand", execute_client_command_handler)
470 | self.server.on_request("workspace/workspaceFolders", workspace_folders_handler)
471 | self.server.on_request("workspace/configuration", workspace_configuration_handler)
472 | self.server.on_notification("$/progress", do_nothing)
473 | self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
474 |
475 | log.info("Starting MATLAB server process")
476 | self.server.start()
477 | initialize_params = self._get_initialize_params(self.repository_root_path)
478 |
479 | log.info("Sending initialize request from LSP client to LSP server and awaiting response")
480 | init_response = self.server.send.initialize(initialize_params)
481 | log.debug(f"Received initialize response from MATLAB server: {init_response}")
482 |
483 | # Verify basic capabilities
484 | capabilities = init_response.get("capabilities", {})
485 | assert capabilities.get("textDocumentSync") in [1, 2], "Expected Full or Incremental text sync"
486 |
487 | # Log available capabilities
488 | if "completionProvider" in capabilities:
489 | log.info("MATLAB server supports completions")
490 | if "definitionProvider" in capabilities:
491 | log.info("MATLAB server supports go-to-definition")
492 | if "referencesProvider" in capabilities:
493 | log.info("MATLAB server supports find-references")
494 | if "documentSymbolProvider" in capabilities:
495 | log.info("MATLAB server supports document symbols")
496 | if "documentFormattingProvider" in capabilities:
497 | log.info("MATLAB server supports document formatting")
498 | if "renameProvider" in capabilities:
499 | log.info("MATLAB server supports rename")
500 |
501 | self.server.notify.initialized({})
502 |
503 | # Wait for server readiness with timeout
504 | # MATLAB takes longer to start than most language servers (typically 10-30 seconds)
505 | log.info("Waiting for MATLAB language server to be ready (this may take up to 60 seconds)...")
506 | if not self.server_ready.wait(timeout=60.0):
507 | # Fallback: assume server is ready after timeout
508 | log.info("Timeout waiting for MATLAB server ready signal, proceeding anyway")
509 | self.server_ready.set()
510 | self.completions_available.set()
511 | else:
512 | log.info("MATLAB server initialization complete")
513 |
514 | def is_ignored_dirname(self, dirname: str) -> bool:
515 | """Define MATLAB-specific directories to ignore."""
516 | return super().is_ignored_dirname(dirname) or dirname in [
517 | "slprj", # Simulink project files
518 | "codegen", # Code generation output
519 | "sldemo_cache", # Simulink demo cache
520 | "helperFiles", # Common helper file directories
521 | ]
522 |
523 | def _request_document_symbols(
524 | self, relative_file_path: str, file_data: LSPFileBuffer | None
525 | ) -> list[SymbolInformation] | list[DocumentSymbol] | None:
526 | """
527 | Override to normalize MATLAB symbol names.
528 |
529 | The MATLAB LSP sometimes returns symbol names as lists instead of strings,
530 | particularly for script sections (cell mode markers like %%). This method
531 | normalizes the names to strings for compatibility with the unified symbol format.
532 | """
533 | symbols = super()._request_document_symbols(relative_file_path, file_data)
534 |
535 | if symbols is None or len(symbols) == 0:
536 | return symbols
537 |
538 | self._normalize_matlab_symbols(symbols)
539 | return symbols
540 |
541 | def _normalize_matlab_symbols(self, symbols: list[SymbolInformation] | list[DocumentSymbol]) -> None:
542 | """
543 | Normalize MATLAB symbol names in-place.
544 |
545 | MATLAB LSP returns section names as lists like ["Section Name"] instead of
546 | strings. This converts them to plain strings.
547 | """
548 | for symbol in symbols:
549 | # MATLAB LSP returns names as lists for script sections, violating LSP spec
550 | # Cast to Any to handle runtime type that differs from spec
551 | name: Any = symbol.get("name")
552 | if isinstance(name, list):
553 | symbol["name"] = name[0] if name else ""
554 | log.debug("Normalized MATLAB symbol name from list to string")
555 |
556 | # Recursively normalize children if present
557 | children: Any = symbol.get("children")
558 | if children and isinstance(children, list):
559 | self._normalize_matlab_symbols(children)
560 |
```
--------------------------------------------------------------------------------
/src/solidlsp/ls_handler.py:
--------------------------------------------------------------------------------
```python
1 | import asyncio
2 | import json
3 | import logging
4 | import os
5 | import platform
6 | import subprocess
7 | import threading
8 | import time
9 | from collections.abc import Callable
10 | from dataclasses import dataclass
11 | from queue import Empty, Queue
12 | from typing import Any
13 |
14 | import psutil
15 | from sensai.util.string import ToStringMixin
16 |
17 | from solidlsp.ls_config import Language
18 | from solidlsp.ls_exceptions import SolidLSPException
19 | from solidlsp.ls_request import LanguageServerRequest
20 | from solidlsp.lsp_protocol_handler.lsp_requests import LspNotification
21 | from solidlsp.lsp_protocol_handler.lsp_types import ErrorCodes
22 | from solidlsp.lsp_protocol_handler.server import (
23 | ENCODING,
24 | LSPError,
25 | MessageType,
26 | PayloadLike,
27 | ProcessLaunchInfo,
28 | StringDict,
29 | content_length,
30 | create_message,
31 | make_error_response,
32 | make_notification,
33 | make_request,
34 | make_response,
35 | )
36 | from solidlsp.util.subprocess_util import quote_arg, subprocess_kwargs
37 |
38 | log = logging.getLogger(__name__)
39 |
40 |
41 | class LanguageServerTerminatedException(Exception):
42 | """
43 | Exception raised when the language server process has terminated unexpectedly.
44 | """
45 |
46 | def __init__(self, message: str, language: Language, cause: Exception | None = None) -> None:
47 | super().__init__(message)
48 | self.message = message
49 | self.language = language
50 | self.cause = cause
51 |
52 | def __str__(self) -> str:
53 | return f"LanguageServerTerminatedException: {self.message}" + (f"; Cause: {self.cause}" if self.cause else "")
54 |
55 |
56 | class Request(ToStringMixin):
57 | @dataclass
58 | class Result:
59 | payload: PayloadLike | None = None
60 | error: Exception | None = None
61 |
62 | def is_error(self) -> bool:
63 | return self.error is not None
64 |
65 | def __init__(self, request_id: int, method: str) -> None:
66 | self._request_id = request_id
67 | self._method = method
68 | self._status = "pending"
69 | self._result_queue: Queue[Request.Result] = Queue()
70 |
71 | def _tostring_includes(self) -> list[str]:
72 | return ["_request_id", "_status", "_method"]
73 |
74 | def on_result(self, params: PayloadLike) -> None:
75 | self._status = "completed"
76 | self._result_queue.put(Request.Result(payload=params))
77 |
78 | def on_error(self, err: Exception) -> None:
79 | """
80 | :param err: the error that occurred while processing the request (typically an LSPError
81 | for errors returned by the LS or LanguageServerTerminatedException if the error
82 | is due to the language server process terminating unexpectedly).
83 | """
84 | self._status = "error"
85 | self._result_queue.put(Request.Result(error=err))
86 |
87 | def get_result(self, timeout: float | None = None) -> Result:
88 | try:
89 | return self._result_queue.get(timeout=timeout)
90 | except Empty as e:
91 | if timeout is not None:
92 | raise TimeoutError(f"Request timed out ({timeout=})") from e
93 | raise e
94 |
95 |
96 | class SolidLanguageServerHandler:
97 | """
98 | This class provides the implementation of Python client for the Language Server Protocol.
99 | A class that launches the language server and communicates with it
100 | using the Language Server Protocol (LSP).
101 |
102 | It provides methods for sending requests, responses, and notifications to the server
103 | and for registering handlers for requests and notifications from the server.
104 |
105 | Uses JSON-RPC 2.0 for communication with the server over stdin/stdout.
106 |
107 | Attributes:
108 | send: A LspRequest object that can be used to send requests to the server and
109 | await for the responses.
110 | notify: A LspNotification object that can be used to send notifications to the server.
111 | cmd: A string that represents the command to launch the language server process.
112 | process: A subprocess.Popen object that represents the language server process.
113 | request_id: An integer that represents the next available request id for the client.
114 | _pending_requests: A dictionary that maps request ids to Request objects that
115 | store the results or errors of the requests.
116 | on_request_handlers: A dictionary that maps method names to callback functions
117 | that handle requests from the server.
118 | on_notification_handlers: A dictionary that maps method names to callback functions
119 | that handle notifications from the server.
120 | logger: An optional function that takes two strings (source and destination) and
121 | a payload dictionary, and logs the communication between the client and the server.
122 | tasks: A dictionary that maps task ids to asyncio.Task objects that represent
123 | the asynchronous tasks created by the handler.
124 | task_counter: An integer that represents the next available task id for the handler.
125 | loop: An asyncio.AbstractEventLoop object that represents the event loop used by the handler.
126 | start_independent_lsp_process: An optional boolean flag that indicates whether to start the
127 | language server process in an independent process group. Default is `True`. Setting it to
128 | `False` means that the language server process will be in the same process group as the
129 | the current process, and any SIGINT and SIGTERM signals will be sent to both processes.
130 |
131 | """
132 |
133 | def __init__(
134 | self,
135 | process_launch_info: ProcessLaunchInfo,
136 | language: Language,
137 | determine_log_level: Callable[[str], int],
138 | logger: Callable[[str, str, StringDict | str], None] | None = None,
139 | start_independent_lsp_process: bool = True,
140 | request_timeout: float | None = None,
141 | ) -> None:
142 | self.language = language
143 | self._determine_log_level = determine_log_level
144 | self.send = LanguageServerRequest(self)
145 | self.notify = LspNotification(self.send_notification)
146 |
147 | self.process_launch_info = process_launch_info
148 | self.process: subprocess.Popen[bytes] | None = None
149 | self._is_shutting_down = False
150 |
151 | self.request_id = 1
152 | self._pending_requests: dict[Any, Request] = {}
153 | self.on_request_handlers: dict[str, Callable[[Any], Any]] = {}
154 | self.on_notification_handlers: dict[str, Callable[[Any], None]] = {}
155 | self.logger = logger
156 | self.tasks: dict[int, Any] = {}
157 | self.task_counter = 0
158 | self.loop = None
159 | self.start_independent_lsp_process = start_independent_lsp_process
160 | self._request_timeout = request_timeout
161 |
162 | # Add thread locks for shared resources to prevent race conditions
163 | self._stdin_lock = threading.Lock()
164 | self._request_id_lock = threading.Lock()
165 | self._response_handlers_lock = threading.Lock()
166 | self._tasks_lock = threading.Lock()
167 |
168 | def set_request_timeout(self, timeout: float | None) -> None:
169 | """
170 | :param timeout: the timeout, in seconds, for all requests sent to the language server.
171 | """
172 | self._request_timeout = timeout
173 |
174 | def is_running(self) -> bool:
175 | """
176 | Checks if the language server process is currently running.
177 | """
178 | return self.process is not None and self.process.returncode is None
179 |
180 | def start(self) -> None:
181 | """
182 | Starts the language server process and creates a task to continuously read from its stdout to handle communications
183 | from the server to the client
184 | """
185 | child_proc_env = os.environ.copy()
186 | child_proc_env.update(self.process_launch_info.env)
187 |
188 | cmd = self.process_launch_info.cmd
189 | is_windows = platform.system() == "Windows"
190 | if not isinstance(cmd, str) and not is_windows:
191 | # Since we are using the shell, we need to convert the command list to a single string
192 | # on Linux/macOS
193 | cmd = " ".join(map(quote_arg, cmd))
194 | log.info("Starting language server process via command: %s", self.process_launch_info.cmd)
195 | kwargs = subprocess_kwargs()
196 | kwargs["start_new_session"] = self.start_independent_lsp_process
197 | self.process = subprocess.Popen(
198 | cmd,
199 | stdout=subprocess.PIPE,
200 | stdin=subprocess.PIPE,
201 | stderr=subprocess.PIPE,
202 | env=child_proc_env,
203 | cwd=self.process_launch_info.cwd,
204 | shell=True,
205 | **kwargs,
206 | )
207 |
208 | # Check if process terminated immediately
209 | if self.process.returncode is not None:
210 | log.error("Language server has already terminated/could not be started")
211 | # Process has already terminated
212 | stderr_data = self.process.stderr.read() if self.process.stderr else b""
213 | error_message = stderr_data.decode("utf-8", errors="replace")
214 | raise RuntimeError(f"Process terminated immediately with code {self.process.returncode}. Error: {error_message}")
215 |
216 | # start threads to read stdout and stderr of the process
217 | threading.Thread(
218 | target=self._read_ls_process_stdout,
219 | name=f"LSP-stdout-reader:{self.language.value}",
220 | daemon=True,
221 | ).start()
222 | threading.Thread(
223 | target=self._read_ls_process_stderr,
224 | name=f"LSP-stderr-reader:{self.language.value}",
225 | daemon=True,
226 | ).start()
227 |
228 | def stop(self) -> None:
229 | """
230 | Sends the terminate signal to the language server process and waits for it to exit, with a timeout, killing it if necessary
231 | """
232 | process = self.process
233 | self.process = None
234 | if process:
235 | self._cleanup_process(process)
236 |
237 | def _cleanup_process(self, process: subprocess.Popen[bytes]) -> None:
238 | """Clean up a process: close stdin, terminate/kill process, close stdout/stderr."""
239 | # Close stdin first to prevent deadlocks
240 | # See: https://bugs.python.org/issue35539
241 | self._safely_close_pipe(process.stdin)
242 |
243 | # Terminate/kill the process if it's still running
244 | if process.returncode is None:
245 | self._terminate_or_kill_process(process)
246 |
247 | # Close stdout and stderr pipes after process has exited
248 | # This is essential to prevent "I/O operation on closed pipe" errors and
249 | # "Event loop is closed" errors during garbage collection
250 | # See: https://bugs.python.org/issue41320 and https://github.com/python/cpython/issues/88050
251 | self._safely_close_pipe(process.stdout)
252 | self._safely_close_pipe(process.stderr)
253 |
254 | def _safely_close_pipe(self, pipe: Any) -> None:
255 | """Safely close a pipe, ignoring any exceptions."""
256 | if pipe:
257 | try:
258 | pipe.close()
259 | except Exception:
260 | pass
261 |
262 | def _terminate_or_kill_process(self, process: subprocess.Popen[bytes]) -> None:
263 | """Try to terminate the process gracefully, then forcefully if necessary."""
264 | # First try to terminate the process tree gracefully
265 | self._signal_process_tree(process, terminate=True)
266 |
267 | def _signal_process_tree(self, process: subprocess.Popen[bytes], terminate: bool = True) -> None:
268 | """Send signal (terminate or kill) to the process and all its children."""
269 | signal_method = "terminate" if terminate else "kill"
270 |
271 | # Try to get the parent process
272 | parent = None
273 | try:
274 | parent = psutil.Process(process.pid)
275 | except (psutil.NoSuchProcess, psutil.AccessDenied, Exception):
276 | pass
277 |
278 | # If we have the parent process and it's running, signal the entire tree
279 | if parent and parent.is_running():
280 | # Signal children first
281 | for child in parent.children(recursive=True):
282 | try:
283 | getattr(child, signal_method)()
284 | except (psutil.NoSuchProcess, psutil.AccessDenied, Exception):
285 | pass
286 |
287 | # Then signal the parent
288 | try:
289 | getattr(parent, signal_method)()
290 | except (psutil.NoSuchProcess, psutil.AccessDenied, Exception):
291 | pass
292 | else:
293 | # Fall back to direct process signaling
294 | try:
295 | getattr(process, signal_method)()
296 | except Exception:
297 | pass
298 |
299 | def shutdown(self) -> None:
300 | """
301 | Perform the shutdown sequence for the client, including sending the shutdown request to the server and notifying it of exit
302 | """
303 | self._is_shutting_down = True
304 | self._log("Sending shutdown request to server")
305 | self.send.shutdown()
306 | self._log("Received shutdown response from server")
307 | self._log("Sending exit notification to server")
308 | self.notify.exit()
309 | self._log("Sent exit notification to server")
310 |
311 | def _log(self, message: str | StringDict) -> None:
312 | """
313 | Create a log message
314 | """
315 | if self.logger is not None:
316 | self.logger("client", "logger", message)
317 |
318 | def _read_bytes_from_process(self, process, stream, num_bytes) -> bytes: # type: ignore
319 | """Read exactly num_bytes from process stdout"""
320 | data = b""
321 | while len(data) < num_bytes:
322 | chunk = stream.read(num_bytes - len(data))
323 | if not chunk:
324 | if process.poll() is not None:
325 | raise LanguageServerTerminatedException(
326 | f"Process terminated while trying to read response (read {num_bytes} of {len(data)} bytes before termination)",
327 | language=self.language,
328 | )
329 | # Process still running but no data available yet, retry after a short delay
330 | time.sleep(0.01)
331 | continue
332 | data += chunk
333 | return data
334 |
335 | def _read_ls_process_stdout(self) -> None:
336 | """
337 | Continuously read from the language server process stdout and handle the messages
338 | invoking the registered response and notification handlers
339 | """
340 | exception: Exception | None = None
341 | try:
342 | while self.process and self.process.stdout:
343 | if self.process.poll() is not None: # process has terminated
344 | break
345 | line = self.process.stdout.readline()
346 | if not line:
347 | continue
348 | try:
349 | num_bytes = content_length(line)
350 | except ValueError:
351 | continue
352 | if num_bytes is None:
353 | continue
354 | while line and line.strip():
355 | line = self.process.stdout.readline()
356 | if not line:
357 | continue
358 | body = self._read_bytes_from_process(self.process, self.process.stdout, num_bytes)
359 |
360 | self._handle_body(body)
361 | except LanguageServerTerminatedException as e:
362 | exception = e
363 | except (BrokenPipeError, ConnectionResetError) as e:
364 | exception = LanguageServerTerminatedException("Language server process terminated while reading stdout", self.language, cause=e)
365 | except Exception as e:
366 | exception = LanguageServerTerminatedException(
367 | "Unexpected error while reading stdout from language server process", self.language, cause=e
368 | )
369 | log.info("Language server stdout reader thread has terminated")
370 | if not self._is_shutting_down:
371 | if exception is None:
372 | exception = LanguageServerTerminatedException("Language server stdout read process terminated unexpectedly", self.language)
373 | log.error(str(exception))
374 | self._cancel_pending_requests(exception)
375 |
376 | def _read_ls_process_stderr(self) -> None:
377 | """
378 | Continuously read from the language server process stderr and log the messages
379 | """
380 | try:
381 | while self.process and self.process.stderr:
382 | if self.process.poll() is not None:
383 | # process has terminated
384 | break
385 | line = self.process.stderr.readline()
386 | if not line:
387 | continue
388 | line_str = line.decode(ENCODING, errors="replace")
389 | level = self._determine_log_level(line_str)
390 | log.log(level, line_str)
391 | except Exception as e:
392 | log.error("Error while reading stderr from language server process: %s", e, exc_info=e)
393 | if not self._is_shutting_down:
394 | log.error("Language server stderr reader thread terminated unexpectedly")
395 | else:
396 | log.info("Language server stderr reader thread has terminated")
397 |
398 | def _handle_body(self, body: bytes) -> None:
399 | """
400 | Parse the body text received from the language server process and invoke the appropriate handler
401 | """
402 | try:
403 | self._receive_payload(json.loads(body))
404 | except OSError as ex:
405 | self._log(f"malformed {ENCODING}: {ex}")
406 | except UnicodeDecodeError as ex:
407 | self._log(f"malformed {ENCODING}: {ex}")
408 | except json.JSONDecodeError as ex:
409 | self._log(f"malformed JSON: {ex}")
410 |
411 | def _receive_payload(self, payload: StringDict) -> None:
412 | """
413 | Determine if the payload received from server is for a request, response, or notification and invoke the appropriate handler
414 | """
415 | if self.logger:
416 | self.logger("server", "client", payload)
417 | try:
418 | if "method" in payload:
419 | if "id" in payload:
420 | self._request_handler(payload)
421 | else:
422 | self._notification_handler(payload)
423 | elif "id" in payload:
424 | self._response_handler(payload)
425 | else:
426 | self._log(f"Unknown payload type: {payload}")
427 | except Exception as err:
428 | self._log(f"Error handling server payload: {err}")
429 |
430 | def send_notification(self, method: str, params: dict | None = None) -> None:
431 | """
432 | Send notification pertaining to the given method to the server with the given parameters
433 | """
434 | self._send_payload(make_notification(method, params))
435 |
436 | def send_response(self, request_id: Any, params: PayloadLike) -> None:
437 | """
438 | Send response to the given request id to the server with the given parameters
439 | """
440 | self._send_payload(make_response(request_id, params))
441 |
442 | def send_error_response(self, request_id: Any, err: LSPError) -> None:
443 | """
444 | Send error response to the given request id to the server with the given error
445 | """
446 | # Use lock to prevent race conditions on tasks and task_counter
447 | self._send_payload(make_error_response(request_id, err))
448 |
449 | def _cancel_pending_requests(self, exception: Exception) -> None:
450 | """
451 | Cancel all pending requests by setting their results to an error
452 | """
453 | with self._response_handlers_lock:
454 | log.info("Cancelling %d pending language server requests", len(self._pending_requests))
455 | for request in self._pending_requests.values():
456 | log.info("Cancelling %s", request)
457 | request.on_error(exception)
458 | self._pending_requests.clear()
459 |
460 | def send_request(self, method: str, params: dict | None = None) -> PayloadLike:
461 | """
462 | Send request to the server, register the request id, and wait for the response
463 | """
464 | with self._request_id_lock:
465 | request_id = self.request_id
466 | self.request_id += 1
467 |
468 | request = Request(request_id=request_id, method=method)
469 | log.debug("Starting: %s", request)
470 |
471 | with self._response_handlers_lock:
472 | self._pending_requests[request_id] = request
473 |
474 | self._send_payload(make_request(method, request_id, params))
475 |
476 | self._log(f"Waiting for response to request {method} with params:\n{params}")
477 | result = request.get_result(timeout=self._request_timeout)
478 | log.debug("Completed: %s", request)
479 |
480 | self._log("Processing result")
481 | if result.is_error():
482 | raise SolidLSPException(f"Error processing request {method} with params:\n{params}", cause=result.error) from result.error
483 |
484 | self._log(f"Returning non-error result, which is:\n{result.payload}")
485 | return result.payload
486 |
487 | def _send_payload(self, payload: StringDict) -> None:
488 | """
489 | Send the payload to the server by writing to its stdin asynchronously.
490 | """
491 | if not self.process or not self.process.stdin:
492 | return
493 | self._log(payload)
494 | msg = create_message(payload)
495 |
496 | # Use lock to prevent concurrent writes to stdin that cause buffer corruption
497 | with self._stdin_lock:
498 | try:
499 | self.process.stdin.writelines(msg)
500 | self.process.stdin.flush()
501 | except (BrokenPipeError, ConnectionResetError, OSError) as e:
502 | # Log the error but don't raise to prevent cascading failures
503 | if self.logger:
504 | self.logger("client", "logger", f"Failed to write to stdin: {e}")
505 | return
506 |
507 | def on_request(self, method: str, cb: Callable[[Any], Any]) -> None:
508 | """
509 | Register the callback function to handle requests from the server to the client for the given method
510 | """
511 | self.on_request_handlers[method] = cb
512 |
513 | def on_notification(self, method: str, cb: Callable[[Any], None]) -> None:
514 | """
515 | Register the callback function to handle notifications from the server to the client for the given method
516 | """
517 | self.on_notification_handlers[method] = cb
518 |
519 | def _response_handler(self, response: StringDict) -> None:
520 | """
521 | Handle the response received from the server for a request, using the id to determine the request
522 | """
523 | response_id = response["id"]
524 | with self._response_handlers_lock:
525 | request = self._pending_requests.pop(response_id, None)
526 | if request is None and isinstance(response_id, str) and response_id.isdigit():
527 | request = self._pending_requests.pop(int(response_id), None)
528 |
529 | if request is None: # need to convert response_id to the right type
530 | log.debug("Request interrupted by user or not found for ID %s", response_id)
531 | return
532 |
533 | if "result" in response and "error" not in response:
534 | request.on_result(response["result"])
535 | elif "result" not in response and "error" in response:
536 | request.on_error(LSPError.from_lsp(response["error"]))
537 | else:
538 | request.on_error(LSPError(ErrorCodes.InvalidRequest, ""))
539 |
540 | def _request_handler(self, response: StringDict) -> None:
541 | """
542 | Handle the request received from the server: call the appropriate callback function and return the result
543 | """
544 | method = response.get("method", "")
545 | params = response.get("params")
546 | request_id = response.get("id")
547 | handler = self.on_request_handlers.get(method)
548 | if not handler:
549 | self.send_error_response(
550 | request_id,
551 | LSPError(
552 | ErrorCodes.MethodNotFound,
553 | f"method '{method}' not handled on client.",
554 | ),
555 | )
556 | return
557 | try:
558 | self.send_response(request_id, handler(params))
559 | except LSPError as ex:
560 | self.send_error_response(request_id, ex)
561 | except Exception as ex:
562 | self.send_error_response(request_id, LSPError(ErrorCodes.InternalError, str(ex)))
563 |
564 | def _notification_handler(self, response: StringDict) -> None:
565 | """
566 | Handle the notification received from the server: call the appropriate callback function
567 | """
568 | method = response.get("method", "")
569 | params = response.get("params")
570 | handler = self.on_notification_handlers.get(method)
571 | if not handler:
572 | self._log(f"unhandled {method}")
573 | return
574 | try:
575 | handler(params)
576 | except asyncio.CancelledError:
577 | return
578 | except Exception as ex:
579 | if (not self._is_shutting_down) and self.logger:
580 | self.logger(
581 | "client",
582 | "logger",
583 | str(
584 | {
585 | "type": MessageType.error,
586 | "message": str(ex),
587 | "method": method,
588 | "params": params,
589 | }
590 | ),
591 | )
592 |
```
--------------------------------------------------------------------------------
/test/serena/util/test_file_system.py:
--------------------------------------------------------------------------------
```python
1 | import os
2 | import shutil
3 | import tempfile
4 | from pathlib import Path
5 |
6 | # Assuming the gitignore parser code is in a module named 'gitignore_parser'
7 | from serena.util.file_system import GitignoreParser, GitignoreSpec
8 |
9 |
10 | class TestGitignoreParser:
11 | """Test class for GitignoreParser functionality."""
12 |
13 | def setup_method(self):
14 | """Set up test environment before each test method."""
15 | # Create a temporary directory for testing
16 | self.test_dir = tempfile.mkdtemp()
17 | self.repo_path = Path(self.test_dir)
18 |
19 | # Create test repository structure
20 | self._create_repo_structure()
21 |
22 | def teardown_method(self):
23 | """Clean up test environment after each test method."""
24 | # Remove the temporary directory
25 | shutil.rmtree(self.test_dir)
26 |
27 | def _create_repo_structure(self):
28 | """
29 | Create a test repository structure with multiple gitignore files.
30 |
31 | Structure:
32 | repo/
33 | ├── .gitignore
34 | ├── file1.txt
35 | ├── test.log
36 | ├── src/
37 | │ ├── .gitignore
38 | │ ├── main.py
39 | │ ├── test.log
40 | │ ├── build/
41 | │ │ └── output.o
42 | │ └── lib/
43 | │ ├── .gitignore
44 | │ └── cache.tmp
45 | └── docs/
46 | ├── .gitignore
47 | ├── api.md
48 | └── temp/
49 | └── draft.md
50 | """
51 | # Create directories
52 | (self.repo_path / "src").mkdir()
53 | (self.repo_path / "src" / "build").mkdir()
54 | (self.repo_path / "src" / "lib").mkdir()
55 | (self.repo_path / "docs").mkdir()
56 | (self.repo_path / "docs" / "temp").mkdir()
57 |
58 | # Create files
59 | (self.repo_path / "file1.txt").touch()
60 | (self.repo_path / "test.log").touch()
61 | (self.repo_path / "src" / "main.py").touch()
62 | (self.repo_path / "src" / "test.log").touch()
63 | (self.repo_path / "src" / "build" / "output.o").touch()
64 | (self.repo_path / "src" / "lib" / "cache.tmp").touch()
65 | (self.repo_path / "docs" / "api.md").touch()
66 | (self.repo_path / "docs" / "temp" / "draft.md").touch()
67 |
68 | # Create root .gitignore
69 | root_gitignore = self.repo_path / ".gitignore"
70 | root_gitignore.write_text(
71 | """# Root gitignore
72 | *.log
73 | /build/
74 | """
75 | )
76 |
77 | # Create src/.gitignore
78 | src_gitignore = self.repo_path / "src" / ".gitignore"
79 | src_gitignore.write_text(
80 | """# Source gitignore
81 | *.o
82 | build/
83 | !important.log
84 | """
85 | )
86 |
87 | # Create src/lib/.gitignore (deeply nested)
88 | src_lib_gitignore = self.repo_path / "src" / "lib" / ".gitignore"
89 | src_lib_gitignore.write_text(
90 | """# Library gitignore
91 | *.tmp
92 | *.cache
93 | """
94 | )
95 |
96 | # Create docs/.gitignore
97 | docs_gitignore = self.repo_path / "docs" / ".gitignore"
98 | docs_gitignore.write_text(
99 | """# Docs gitignore
100 | temp/
101 | *.tmp
102 | """
103 | )
104 |
105 | def test_initialization(self):
106 | """Test GitignoreParser initialization."""
107 | parser = GitignoreParser(str(self.repo_path))
108 |
109 | assert parser.repo_root == str(self.repo_path.absolute())
110 | assert len(parser.get_ignore_specs()) == 4
111 |
112 | def test_find_gitignore_files(self):
113 | """Test finding all gitignore files in repository, including deeply nested ones."""
114 | parser = GitignoreParser(str(self.repo_path))
115 |
116 | # Get file paths from specs
117 | gitignore_files = [spec.file_path for spec in parser.get_ignore_specs()]
118 |
119 | # Convert to relative paths for easier testing
120 | rel_paths = [os.path.relpath(f, self.repo_path) for f in gitignore_files]
121 | rel_paths.sort()
122 |
123 | assert len(rel_paths) == 4
124 | assert ".gitignore" in rel_paths
125 | assert os.path.join("src", ".gitignore") in rel_paths
126 | assert os.path.join("src", "lib", ".gitignore") in rel_paths # Deeply nested
127 | assert os.path.join("docs", ".gitignore") in rel_paths
128 |
129 | def test_parse_patterns_root_directory(self):
130 | """Test parsing gitignore patterns in root directory."""
131 | # Create a simple test case with only root gitignore
132 | test_dir = self.repo_path / "test_root"
133 | test_dir.mkdir()
134 |
135 | gitignore = test_dir / ".gitignore"
136 | gitignore.write_text(
137 | """*.log
138 | build/
139 | /temp.txt
140 | """
141 | )
142 |
143 | parser = GitignoreParser(str(test_dir))
144 | specs = parser.get_ignore_specs()
145 |
146 | assert len(specs) == 1
147 | patterns = specs[0].patterns
148 |
149 | assert "*.log" in patterns
150 | assert "build/" in patterns
151 | assert "/temp.txt" in patterns
152 |
153 | def test_parse_patterns_subdirectory(self):
154 | """Test parsing gitignore patterns in subdirectory."""
155 | # Create a test case with subdirectory gitignore
156 | test_dir = self.repo_path / "test_sub"
157 | test_dir.mkdir()
158 | subdir = test_dir / "src"
159 | subdir.mkdir()
160 |
161 | gitignore = subdir / ".gitignore"
162 | gitignore.write_text(
163 | """*.o
164 | /build/
165 | test.log
166 | """
167 | )
168 |
169 | parser = GitignoreParser(str(test_dir))
170 | specs = parser.get_ignore_specs()
171 |
172 | assert len(specs) == 1
173 | patterns = specs[0].patterns
174 |
175 | # Non-anchored pattern should get ** prefix
176 | assert "src/**/*.o" in patterns
177 | # Anchored pattern should not get ** prefix
178 | assert "src/build/" in patterns
179 | # Non-anchored pattern without slash
180 | assert "src/**/test.log" in patterns
181 |
182 | def test_should_ignore_root_patterns(self):
183 | """Test ignoring files based on root .gitignore."""
184 | parser = GitignoreParser(str(self.repo_path))
185 |
186 | # Files that should be ignored
187 | assert parser.should_ignore("test.log")
188 | assert parser.should_ignore(str(self.repo_path / "test.log"))
189 |
190 | # Files that should NOT be ignored
191 | assert not parser.should_ignore("file1.txt")
192 | assert not parser.should_ignore("src/main.py")
193 |
194 | def test_should_ignore_subdirectory_patterns(self):
195 | """Test ignoring files based on subdirectory .gitignore files."""
196 | parser = GitignoreParser(str(self.repo_path))
197 |
198 | # .o files in src should be ignored
199 | assert parser.should_ignore("src/build/output.o")
200 |
201 | # build/ directory in src should be ignored
202 | assert parser.should_ignore("src/build/")
203 |
204 | # temp/ directory in docs should be ignored
205 | assert parser.should_ignore("docs/temp/draft.md")
206 |
207 | # But temp/ outside docs should not be ignored by docs/.gitignore
208 | assert not parser.should_ignore("temp/file.txt")
209 |
210 | # Test deeply nested .gitignore in src/lib/
211 | # .tmp files in src/lib should be ignored
212 | assert parser.should_ignore("src/lib/cache.tmp")
213 |
214 | # .cache files in src/lib should also be ignored
215 | assert parser.should_ignore("src/lib/data.cache")
216 |
217 | # But .tmp files outside src/lib should not be ignored by src/lib/.gitignore
218 | assert not parser.should_ignore("src/other.tmp")
219 |
220 | def test_anchored_vs_non_anchored_patterns(self):
221 | """Test the difference between anchored and non-anchored patterns."""
222 | # Create new test structure
223 | test_dir = self.repo_path / "test_anchored"
224 | test_dir.mkdir()
225 | (test_dir / "src").mkdir()
226 | (test_dir / "src" / "subdir").mkdir()
227 | (test_dir / "src" / "subdir" / "deep").mkdir()
228 |
229 | # Create src/.gitignore with both anchored and non-anchored patterns
230 | gitignore = test_dir / "src" / ".gitignore"
231 | gitignore.write_text(
232 | """/temp.txt
233 | data.json
234 | """
235 | )
236 |
237 | # Create test files
238 | (test_dir / "src" / "temp.txt").touch()
239 | (test_dir / "src" / "data.json").touch()
240 | (test_dir / "src" / "subdir" / "temp.txt").touch()
241 | (test_dir / "src" / "subdir" / "data.json").touch()
242 | (test_dir / "src" / "subdir" / "deep" / "data.json").touch()
243 |
244 | parser = GitignoreParser(str(test_dir))
245 |
246 | # Anchored pattern /temp.txt should only match in src/
247 | assert parser.should_ignore("src/temp.txt")
248 | assert not parser.should_ignore("src/subdir/temp.txt")
249 |
250 | # Non-anchored pattern data.json should match anywhere under src/
251 | assert parser.should_ignore("src/data.json")
252 | assert parser.should_ignore("src/subdir/data.json")
253 | assert parser.should_ignore("src/subdir/deep/data.json")
254 |
255 | def test_root_anchored_patterns(self):
256 | """Test anchored patterns in root .gitignore only match root-level files."""
257 | # Create new test structure for root anchored patterns
258 | test_dir = self.repo_path / "test_root_anchored"
259 | test_dir.mkdir()
260 | (test_dir / "src").mkdir()
261 | (test_dir / "docs").mkdir()
262 | (test_dir / "src" / "nested").mkdir()
263 |
264 | # Create root .gitignore with anchored patterns
265 | gitignore = test_dir / ".gitignore"
266 | gitignore.write_text(
267 | """/config.json
268 | /temp.log
269 | /build
270 | *.pyc
271 | """
272 | )
273 |
274 | # Create test files at root level
275 | (test_dir / "config.json").touch()
276 | (test_dir / "temp.log").touch()
277 | (test_dir / "build").mkdir()
278 | (test_dir / "file.pyc").touch()
279 |
280 | # Create same-named files in subdirectories
281 | (test_dir / "src" / "config.json").touch()
282 | (test_dir / "src" / "temp.log").touch()
283 | (test_dir / "src" / "build").mkdir()
284 | (test_dir / "src" / "file.pyc").touch()
285 | (test_dir / "docs" / "config.json").touch()
286 | (test_dir / "docs" / "temp.log").touch()
287 | (test_dir / "src" / "nested" / "config.json").touch()
288 | (test_dir / "src" / "nested" / "temp.log").touch()
289 | (test_dir / "src" / "nested" / "build").mkdir()
290 |
291 | parser = GitignoreParser(str(test_dir))
292 |
293 | # Anchored patterns should only match root-level files
294 | assert parser.should_ignore("config.json")
295 | assert not parser.should_ignore("src/config.json")
296 | assert not parser.should_ignore("docs/config.json")
297 | assert not parser.should_ignore("src/nested/config.json")
298 |
299 | assert parser.should_ignore("temp.log")
300 | assert not parser.should_ignore("src/temp.log")
301 | assert not parser.should_ignore("docs/temp.log")
302 | assert not parser.should_ignore("src/nested/temp.log")
303 |
304 | assert parser.should_ignore("build")
305 | assert not parser.should_ignore("src/build")
306 | assert not parser.should_ignore("src/nested/build")
307 |
308 | # Non-anchored patterns should match everywhere
309 | assert parser.should_ignore("file.pyc")
310 | assert parser.should_ignore("src/file.pyc")
311 |
312 | def test_mixed_anchored_and_non_anchored_root_patterns(self):
313 | """Test mix of anchored and non-anchored patterns in root .gitignore."""
314 | test_dir = self.repo_path / "test_mixed_patterns"
315 | test_dir.mkdir()
316 | (test_dir / "app").mkdir()
317 | (test_dir / "tests").mkdir()
318 | (test_dir / "app" / "modules").mkdir()
319 |
320 | # Create root .gitignore with mixed patterns
321 | gitignore = test_dir / ".gitignore"
322 | gitignore.write_text(
323 | """/secrets.env
324 | /dist/
325 | node_modules/
326 | *.tmp
327 | /app/local.config
328 | debug.log
329 | """
330 | )
331 |
332 | # Create test files and directories
333 | (test_dir / "secrets.env").touch()
334 | (test_dir / "dist").mkdir()
335 | (test_dir / "node_modules").mkdir()
336 | (test_dir / "file.tmp").touch()
337 | (test_dir / "app" / "local.config").touch()
338 | (test_dir / "debug.log").touch()
339 |
340 | # Create same files in subdirectories
341 | (test_dir / "app" / "secrets.env").touch()
342 | (test_dir / "app" / "dist").mkdir()
343 | (test_dir / "app" / "node_modules").mkdir()
344 | (test_dir / "app" / "file.tmp").touch()
345 | (test_dir / "app" / "debug.log").touch()
346 | (test_dir / "tests" / "secrets.env").touch()
347 | (test_dir / "tests" / "node_modules").mkdir()
348 | (test_dir / "tests" / "debug.log").touch()
349 | (test_dir / "app" / "modules" / "local.config").touch()
350 |
351 | parser = GitignoreParser(str(test_dir))
352 |
353 | # Anchored patterns should only match at root
354 | assert parser.should_ignore("secrets.env")
355 | assert not parser.should_ignore("app/secrets.env")
356 | assert not parser.should_ignore("tests/secrets.env")
357 |
358 | assert parser.should_ignore("dist")
359 | assert not parser.should_ignore("app/dist")
360 |
361 | assert parser.should_ignore("app/local.config")
362 | assert not parser.should_ignore("app/modules/local.config")
363 |
364 | # Non-anchored patterns should match everywhere
365 | assert parser.should_ignore("node_modules")
366 | assert parser.should_ignore("app/node_modules")
367 | assert parser.should_ignore("tests/node_modules")
368 |
369 | assert parser.should_ignore("file.tmp")
370 | assert parser.should_ignore("app/file.tmp")
371 |
372 | assert parser.should_ignore("debug.log")
373 | assert parser.should_ignore("app/debug.log")
374 | assert parser.should_ignore("tests/debug.log")
375 |
376 | def test_negation_patterns(self):
377 | """Test negation patterns are parsed correctly."""
378 | test_dir = self.repo_path / "test_negation"
379 | test_dir.mkdir()
380 |
381 | gitignore = test_dir / ".gitignore"
382 | gitignore.write_text(
383 | """*.log
384 | !important.log
385 | !src/keep.log
386 | """
387 | )
388 |
389 | parser = GitignoreParser(str(test_dir))
390 | specs = parser.get_ignore_specs()
391 |
392 | assert len(specs) == 1
393 | patterns = specs[0].patterns
394 |
395 | assert "*.log" in patterns
396 | assert "!important.log" in patterns
397 | assert "!src/keep.log" in patterns
398 |
399 | def test_comments_and_empty_lines(self):
400 | """Test that comments and empty lines are ignored."""
401 | test_dir = self.repo_path / "test_comments"
402 | test_dir.mkdir()
403 |
404 | gitignore = test_dir / ".gitignore"
405 | gitignore.write_text(
406 | """# This is a comment
407 | *.log
408 |
409 | # Another comment
410 | # Indented comment
411 |
412 | build/
413 | """
414 | )
415 |
416 | parser = GitignoreParser(str(test_dir))
417 | specs = parser.get_ignore_specs()
418 |
419 | assert len(specs) == 1
420 | patterns = specs[0].patterns
421 |
422 | assert len(patterns) == 2
423 | assert "*.log" in patterns
424 | assert "build/" in patterns
425 |
426 | def test_escaped_characters(self):
427 | """Test escaped special characters."""
428 | test_dir = self.repo_path / "test_escaped"
429 | test_dir.mkdir()
430 |
431 | gitignore = test_dir / ".gitignore"
432 | gitignore.write_text(
433 | """\\#not-a-comment.txt
434 | \\!not-negation.txt
435 | """
436 | )
437 |
438 | parser = GitignoreParser(str(test_dir))
439 | specs = parser.get_ignore_specs()
440 |
441 | assert len(specs) == 1
442 | patterns = specs[0].patterns
443 |
444 | assert "#not-a-comment.txt" in patterns
445 | assert "!not-negation.txt" in patterns
446 |
447 | def test_escaped_negation_patterns(self):
448 | test_dir = self.repo_path / "test_escaped_negation"
449 | test_dir.mkdir()
450 |
451 | gitignore = test_dir / ".gitignore"
452 | gitignore.write_text(
453 | """*.log
454 | \\!not-negation.log
455 | !actual-negation.log
456 | """
457 | )
458 |
459 | parser = GitignoreParser(str(test_dir))
460 | specs = parser.get_ignore_specs()
461 |
462 | assert len(specs) == 1
463 | patterns = specs[0].patterns
464 |
465 | # Key assertions: escaped exclamation becomes literal, real negation preserved
466 | assert "!not-negation.log" in patterns # escaped -> literal
467 | assert "!actual-negation.log" in patterns # real negation preserved
468 |
469 | # Test the actual behavioral difference between escaped and real negation:
470 | # *.log pattern should ignore test.log
471 | assert parser.should_ignore("test.log")
472 |
473 | # Escaped negation file should still be ignored by *.log pattern
474 | assert parser.should_ignore("!not-negation.log")
475 |
476 | # Actual negation should override the *.log pattern
477 | assert not parser.should_ignore("actual-negation.log")
478 |
479 | def test_glob_patterns(self):
480 | """Test various glob patterns work correctly."""
481 | test_dir = self.repo_path / "test_glob"
482 | test_dir.mkdir()
483 |
484 | gitignore = test_dir / ".gitignore"
485 | gitignore.write_text(
486 | """*.pyc
487 | **/*.tmp
488 | src/*.o
489 | !src/important.o
490 | [Tt]est*
491 | """
492 | )
493 |
494 | # Create test files
495 | (test_dir / "src").mkdir()
496 | (test_dir / "src" / "nested").mkdir()
497 | (test_dir / "file.pyc").touch()
498 | (test_dir / "src" / "file.pyc").touch()
499 | (test_dir / "file.tmp").touch()
500 | (test_dir / "src" / "nested" / "file.tmp").touch()
501 | (test_dir / "src" / "file.o").touch()
502 | (test_dir / "src" / "important.o").touch()
503 | (test_dir / "Test.txt").touch()
504 | (test_dir / "test.log").touch()
505 |
506 | parser = GitignoreParser(str(test_dir))
507 |
508 | # *.pyc should match everywhere
509 | assert parser.should_ignore("file.pyc")
510 | assert parser.should_ignore("src/file.pyc")
511 |
512 | # **/*.tmp should match all .tmp files
513 | assert parser.should_ignore("file.tmp")
514 | assert parser.should_ignore("src/nested/file.tmp")
515 |
516 | # src/*.o should only match .o files directly in src/
517 | assert parser.should_ignore("src/file.o")
518 |
519 | # Character class patterns
520 | assert parser.should_ignore("Test.txt")
521 | assert parser.should_ignore("test.log")
522 |
523 | def test_empty_gitignore(self):
524 | """Test handling of empty gitignore files."""
525 | test_dir = self.repo_path / "test_empty"
526 | test_dir.mkdir()
527 |
528 | gitignore = test_dir / ".gitignore"
529 | gitignore.write_text("")
530 |
531 | parser = GitignoreParser(str(test_dir))
532 |
533 | # Should not crash and should return empty list
534 | assert len(parser.get_ignore_specs()) == 0
535 |
536 | def test_malformed_gitignore(self):
537 | """Test handling of malformed gitignore content."""
538 | test_dir = self.repo_path / "test_malformed"
539 | test_dir.mkdir()
540 |
541 | gitignore = test_dir / ".gitignore"
542 | gitignore.write_text(
543 | """# Only comments and empty lines
544 |
545 | # More comments
546 |
547 | """
548 | )
549 |
550 | parser = GitignoreParser(str(test_dir))
551 |
552 | # Should handle gracefully
553 | assert len(parser.get_ignore_specs()) == 0
554 |
555 | def test_reload(self):
556 | """Test reloading gitignore files."""
557 | test_dir = self.repo_path / "test_reload"
558 | test_dir.mkdir()
559 |
560 | # Create initial gitignore
561 | gitignore = test_dir / ".gitignore"
562 | gitignore.write_text("*.log")
563 |
564 | parser = GitignoreParser(str(test_dir))
565 | assert len(parser.get_ignore_specs()) == 1
566 | assert parser.should_ignore("test.log")
567 |
568 | # Modify gitignore
569 | gitignore.write_text("*.tmp")
570 |
571 | # Without reload, should still use old patterns
572 | assert parser.should_ignore("test.log")
573 | assert not parser.should_ignore("test.tmp")
574 |
575 | # After reload, should use new patterns
576 | parser.reload()
577 | assert not parser.should_ignore("test.log")
578 | assert parser.should_ignore("test.tmp")
579 |
580 | def test_gitignore_spec_matches(self):
581 | """Test GitignoreSpec.matches method."""
582 | spec = GitignoreSpec("/path/to/.gitignore", ["*.log", "build/", "!important.log"])
583 |
584 | assert spec.matches("test.log")
585 | assert spec.matches("build/output.o")
586 | assert spec.matches("src/test.log")
587 |
588 | # Note: Negation patterns in pathspec work differently than in git
589 | # This is a limitation of the pathspec library
590 |
591 | def test_subdirectory_gitignore_pattern_scoping(self):
592 | """Test that subdirectory .gitignore patterns are scoped correctly."""
593 | # Create test structure: foo/ with subdirectory bar/
594 | test_dir = self.repo_path / "test_subdir_scoping"
595 | test_dir.mkdir()
596 | (test_dir / "foo").mkdir()
597 | (test_dir / "foo" / "bar").mkdir()
598 |
599 | # Create files in various locations
600 | (test_dir / "foo.txt").touch() # root level
601 | (test_dir / "foo" / "foo.txt").touch() # in foo/
602 | (test_dir / "foo" / "bar" / "foo.txt").touch() # in foo/bar/
603 |
604 | # Test case 1: foo.txt in foo/.gitignore should only ignore in foo/ subtree
605 | gitignore = test_dir / "foo" / ".gitignore"
606 | gitignore.write_text("foo.txt\n")
607 |
608 | parser = GitignoreParser(str(test_dir))
609 |
610 | # foo.txt at root should NOT be ignored by foo/.gitignore
611 | assert not parser.should_ignore("foo.txt"), "Root foo.txt should not be ignored by foo/.gitignore"
612 |
613 | # foo.txt in foo/ should be ignored
614 | assert parser.should_ignore("foo/foo.txt"), "foo/foo.txt should be ignored"
615 |
616 | # foo.txt in foo/bar/ should be ignored (within foo/ subtree)
617 | assert parser.should_ignore("foo/bar/foo.txt"), "foo/bar/foo.txt should be ignored"
618 |
619 | def test_anchored_pattern_in_subdirectory(self):
620 | """Test that anchored patterns in subdirectory only match immediate children."""
621 | test_dir = self.repo_path / "test_anchored_subdir"
622 | test_dir.mkdir()
623 | (test_dir / "foo").mkdir()
624 | (test_dir / "foo" / "bar").mkdir()
625 |
626 | # Create files
627 | (test_dir / "foo.txt").touch() # root level
628 | (test_dir / "foo" / "foo.txt").touch() # in foo/
629 | (test_dir / "foo" / "bar" / "foo.txt").touch() # in foo/bar/
630 |
631 | # Test case 2: /foo.txt in foo/.gitignore should only match foo/foo.txt
632 | gitignore = test_dir / "foo" / ".gitignore"
633 | gitignore.write_text("/foo.txt\n")
634 |
635 | parser = GitignoreParser(str(test_dir))
636 |
637 | # foo.txt at root should NOT be ignored
638 | assert not parser.should_ignore("foo.txt"), "Root foo.txt should not be ignored"
639 |
640 | # foo.txt directly in foo/ should be ignored
641 | assert parser.should_ignore("foo/foo.txt"), "foo/foo.txt should be ignored by /foo.txt pattern"
642 |
643 | # foo.txt in foo/bar/ should NOT be ignored (anchored pattern only matches immediate children)
644 | assert not parser.should_ignore("foo/bar/foo.txt"), "foo/bar/foo.txt should NOT be ignored by /foo.txt pattern"
645 |
646 | def test_double_star_pattern_scoping(self):
647 | """Test that **/pattern in subdirectory only applies within that subtree."""
648 | test_dir = self.repo_path / "test_doublestar_scope"
649 | test_dir.mkdir()
650 | (test_dir / "foo").mkdir()
651 | (test_dir / "foo" / "bar").mkdir()
652 | (test_dir / "other").mkdir()
653 |
654 | # Create files
655 | (test_dir / "foo.txt").touch() # root level
656 | (test_dir / "foo" / "foo.txt").touch() # in foo/
657 | (test_dir / "foo" / "bar" / "foo.txt").touch() # in foo/bar/
658 | (test_dir / "other" / "foo.txt").touch() # in other/
659 |
660 | # Test case 3: **/foo.txt in foo/.gitignore should only ignore within foo/ subtree
661 | gitignore = test_dir / "foo" / ".gitignore"
662 | gitignore.write_text("**/foo.txt\n")
663 |
664 | parser = GitignoreParser(str(test_dir))
665 |
666 | # foo.txt at root should NOT be ignored
667 | assert not parser.should_ignore("foo.txt"), "Root foo.txt should not be ignored by foo/.gitignore"
668 |
669 | # foo.txt in foo/ should be ignored
670 | assert parser.should_ignore("foo/foo.txt"), "foo/foo.txt should be ignored"
671 |
672 | # foo.txt in foo/bar/ should be ignored (within foo/ subtree)
673 | assert parser.should_ignore("foo/bar/foo.txt"), "foo/bar/foo.txt should be ignored"
674 |
675 | # foo.txt in other/ should NOT be ignored (outside foo/ subtree)
676 | assert not parser.should_ignore("other/foo.txt"), "other/foo.txt should NOT be ignored by foo/.gitignore"
677 |
678 | def test_anchored_double_star_pattern(self):
679 | """Test that /**/pattern in subdirectory works correctly."""
680 | test_dir = self.repo_path / "test_anchored_doublestar"
681 | test_dir.mkdir()
682 | (test_dir / "foo").mkdir()
683 | (test_dir / "foo" / "bar").mkdir()
684 | (test_dir / "other").mkdir()
685 |
686 | # Create files
687 | (test_dir / "foo.txt").touch() # root level
688 | (test_dir / "foo" / "foo.txt").touch() # in foo/
689 | (test_dir / "foo" / "bar" / "foo.txt").touch() # in foo/bar/
690 | (test_dir / "other" / "foo.txt").touch() # in other/
691 |
692 | # Test case 4: /**/foo.txt in foo/.gitignore should correctly ignore only within foo/ subtree
693 | gitignore = test_dir / "foo" / ".gitignore"
694 | gitignore.write_text("/**/foo.txt\n")
695 |
696 | parser = GitignoreParser(str(test_dir))
697 |
698 | # foo.txt at root should NOT be ignored
699 | assert not parser.should_ignore("foo.txt"), "Root foo.txt should not be ignored"
700 |
701 | # foo.txt in foo/ should be ignored
702 | assert parser.should_ignore("foo/foo.txt"), "foo/foo.txt should be ignored"
703 |
704 | # foo.txt in foo/bar/ should be ignored (within foo/ subtree)
705 | assert parser.should_ignore("foo/bar/foo.txt"), "foo/bar/foo.txt should be ignored"
706 |
707 | # foo.txt in other/ should NOT be ignored (outside foo/ subtree)
708 | assert not parser.should_ignore("other/foo.txt"), "other/foo.txt should NOT be ignored by foo/.gitignore"
709 |
```
--------------------------------------------------------------------------------
/test/serena/test_text_utils.py:
--------------------------------------------------------------------------------
```python
1 | import re
2 |
3 | import pytest
4 |
5 | from serena.text_utils import LineType, search_files, search_text
6 |
7 |
8 | class TestSearchText:
9 | def test_search_text_with_string_pattern(self):
10 | """Test searching with a simple string pattern."""
11 | content = """
12 | def hello_world():
13 | print("Hello, World!")
14 | return 42
15 | """
16 |
17 | # Search for a simple string pattern
18 | matches = search_text("print", content=content)
19 |
20 | assert len(matches) == 1
21 | assert matches[0].num_matched_lines == 1
22 | assert matches[0].start_line == 3
23 | assert matches[0].end_line == 3
24 | assert matches[0].lines[0].line_content.strip() == 'print("Hello, World!")'
25 |
26 | def test_search_text_with_regex_pattern(self):
27 | """Test searching with a regex pattern."""
28 | content = """
29 | class DataProcessor:
30 | def __init__(self, data):
31 | self.data = data
32 |
33 | def process(self):
34 | return [x * 2 for x in self.data if x > 0]
35 |
36 | def filter(self, predicate):
37 | return [x for x in self.data if predicate(x)]
38 | """
39 |
40 | # Search for a regex pattern matching method definitions
41 | pattern = r"def\s+\w+\s*\([^)]*\):"
42 | matches = search_text(pattern, content=content)
43 |
44 | assert len(matches) == 3
45 | assert matches[0].lines[0].match_type == LineType.MATCH
46 | assert "def __init__" in matches[0].lines[0].line_content
47 | assert "def process" in matches[1].lines[0].line_content
48 | assert "def filter" in matches[2].lines[0].line_content
49 |
50 | def test_search_text_with_compiled_regex(self):
51 | """Test searching with a pre-compiled regex pattern."""
52 | content = """
53 | import os
54 | import sys
55 | from pathlib import Path
56 |
57 | # Configuration variables
58 | DEBUG = True
59 | MAX_RETRIES = 3
60 |
61 | def configure_logging():
62 | log_level = "DEBUG" if DEBUG else "INFO"
63 | print(f"Setting log level to {log_level}")
64 | """
65 |
66 | # Search for variable assignments with a compiled regex
67 | pattern = re.compile(r"^\s*[A-Z_]+ = .+$")
68 | matches = search_text(pattern, content=content)
69 |
70 | assert len(matches) == 2
71 | assert "DEBUG = True" in matches[0].lines[0].line_content
72 | assert "MAX_RETRIES = 3" in matches[1].lines[0].line_content
73 |
74 | def test_search_text_with_context_lines(self):
75 | """Test searching with context lines before and after the match."""
76 | content = """
77 | def complex_function(a, b, c):
78 | # This is a complex function that does something.
79 | if a > b:
80 | return a * c
81 | elif b > a:
82 | return b * c
83 | else:
84 | return (a + b) * c
85 | """
86 |
87 | # Search with context lines
88 | matches = search_text("return", content=content, context_lines_before=1, context_lines_after=1)
89 |
90 | assert len(matches) == 3
91 |
92 | # Check the first match with context
93 | first_match = matches[0]
94 | assert len(first_match.lines) == 3
95 | assert first_match.lines[0].match_type == LineType.BEFORE_MATCH
96 | assert first_match.lines[1].match_type == LineType.MATCH
97 | assert first_match.lines[2].match_type == LineType.AFTER_MATCH
98 |
99 | # Verify the content of lines
100 | assert "if a > b:" in first_match.lines[0].line_content
101 | assert "return a * c" in first_match.lines[1].line_content
102 | assert "elif b > a:" in first_match.lines[2].line_content
103 |
104 | def test_search_text_with_multiline_match(self):
105 | """Test searching with multiline pattern matching."""
106 | content = """
107 | def factorial(n):
108 | if n <= 1:
109 | return 1
110 | else:
111 | return n * factorial(n-1)
112 |
113 | result = factorial(5) # Should be 120
114 | """
115 |
116 | # Search for a pattern that spans multiple lines (if-else block)
117 | pattern = r"if.*?else.*?return"
118 | matches = search_text(pattern, content=content, allow_multiline_match=True)
119 |
120 | assert len(matches) == 1
121 | multiline_match = matches[0]
122 | assert multiline_match.num_matched_lines >= 3
123 | assert "if n <= 1:" in multiline_match.lines[0].line_content
124 |
125 | # All matched lines should have match_type == LineType.MATCH
126 | match_lines = [line for line in multiline_match.lines if line.match_type == LineType.MATCH]
127 | assert len(match_lines) >= 3
128 |
129 | def test_search_text_with_glob_pattern(self):
130 | """Test searching with glob-like patterns."""
131 | content = """
132 | class UserService:
133 | def get_user(self, user_id):
134 | return {"id": user_id, "name": "Test User"}
135 |
136 | def create_user(self, user_data):
137 | print(f"Creating user: {user_data}")
138 | return {"id": 123, **user_data}
139 |
140 | def update_user(self, user_id, user_data):
141 | print(f"Updating user {user_id} with {user_data}")
142 | return True
143 | """
144 |
145 | # Search with a glob pattern for all user methods
146 | matches = search_text("*_user*", content=content, is_glob=True)
147 |
148 | assert len(matches) == 3
149 | assert "get_user" in matches[0].lines[0].line_content
150 | assert "create_user" in matches[1].lines[0].line_content
151 | assert "update_user" in matches[2].lines[0].line_content
152 |
153 | def test_search_text_with_complex_glob_pattern(self):
154 | """Test searching with more complex glob patterns."""
155 | content = """
156 | def process_data(data):
157 | return [transform(item) for item in data]
158 |
159 | def transform(item):
160 | if isinstance(item, dict):
161 | return {k: v.upper() if isinstance(v, str) else v for k, v in item.items()}
162 | elif isinstance(item, list):
163 | return [x * 2 for x in item if isinstance(x, (int, float))]
164 | elif isinstance(item, str):
165 | return item.upper()
166 | else:
167 | return item
168 | """
169 |
170 | # Search with a simplified glob pattern to find all isinstance occurrences
171 | matches = search_text("*isinstance*", content=content, is_glob=True)
172 |
173 | # Should match lines with isinstance(item, dict) and isinstance(item, list)
174 | assert len(matches) >= 2
175 | instance_matches = [
176 | line.line_content
177 | for match in matches
178 | for line in match.lines
179 | if line.match_type == LineType.MATCH and "isinstance(item," in line.line_content
180 | ]
181 | assert len(instance_matches) >= 2
182 | assert any("isinstance(item, dict)" in line for line in instance_matches)
183 | assert any("isinstance(item, list)" in line for line in instance_matches)
184 |
185 | def test_search_text_glob_with_special_chars(self):
186 | """Glob patterns containing regex special characters should match literally."""
187 | content = """
188 | def func_square():
189 | print("value[42]")
190 |
191 | def func_curly():
192 | print("value{bar}")
193 | """
194 |
195 | matches_square = search_text(r"*\[42\]*", content=content, is_glob=True)
196 | assert len(matches_square) == 1
197 | assert "[42]" in matches_square[0].lines[0].line_content
198 |
199 | matches_curly = search_text("*{bar}*", content=content, is_glob=True)
200 | assert len(matches_curly) == 1
201 | assert "{bar}" in matches_curly[0].lines[0].line_content
202 |
203 | def test_search_text_no_matches(self):
204 | """Test searching with a pattern that doesn't match anything."""
205 | content = """
206 | def calculate_average(numbers):
207 | if not numbers:
208 | return 0
209 | return sum(numbers) / len(numbers)
210 | """
211 |
212 | # Search for a pattern that doesn't exist in the content
213 | matches = search_text("missing_function", content=content)
214 |
215 | assert len(matches) == 0
216 |
217 |
218 | # Mock file reader that always returns matching content
219 | def mock_reader_always_match(file_path: str) -> str:
220 | """Mock file reader that returns content guaranteed to match the simple pattern."""
221 | return "This line contains a match."
222 |
223 |
224 | class TestSearchFiles:
225 | @pytest.mark.parametrize(
226 | "file_paths, pattern, paths_include_glob, paths_exclude_glob, expected_matched_files, description",
227 | [
228 | # Basic cases
229 | (["a.py", "b.txt"], "match", None, None, ["a.py", "b.txt"], "No filters"),
230 | (["a.py", "b.txt"], "match", "*.py", None, ["a.py"], "Include only .py files"),
231 | (["a.py", "b.txt"], "match", None, "*.txt", ["a.py"], "Exclude .txt files"),
232 | (["a.py", "b.txt", "c.py"], "match", "*.py", "c.*", ["a.py"], "Include .py, exclude c.*"),
233 | # Directory matching - Using pathspec patterns
234 | (["main.c", "test/main.c"], "match", "test/*", None, ["test/main.c"], "Include files in test/ subdir"),
235 | (["data/a.csv", "data/b.log"], "match", "data/*", "*.log", ["data/a.csv"], "Include data/*, exclude *.log"),
236 | (["src/a.py", "tests/b.py"], "match", "src/**", "tests/**", ["src/a.py"], "Include src/**, exclude tests/**"),
237 | (["src/mod/a.py", "tests/b.py"], "match", "**/*.py", "tests/**", ["src/mod/a.py"], "Include **/*.py, exclude tests/**"),
238 | (["file.py", "dir/file.py"], "match", "dir/*.py", None, ["dir/file.py"], "Include files directly in dir"),
239 | (["file.py", "dir/sub/file.py"], "match", "dir/**/*.py", None, ["dir/sub/file.py"], "Include files recursively in dir"),
240 | # Overlap and edge cases
241 | (["file.py", "dir/file.py"], "match", "*.py", "dir/*", ["file.py"], "Include *.py, exclude files directly in dir"),
242 | (["root.py", "adir/a.py", "bdir/b.py"], "match", "a*/*.py", None, ["adir/a.py"], "Include files in dirs starting with 'a'"),
243 | (["a.txt", "b.log"], "match", "*.py", None, [], "No files match include pattern"),
244 | (["a.py", "b.py"], "match", None, "*.py", [], "All files match exclude pattern"),
245 | (["a.py", "b.py"], "match", "a.*", "*.py", [], "Include a.* but exclude *.py -> empty"),
246 | (["a.py", "b.py"], "match", "*.py", "b.*", ["a.py"], "Include *.py but exclude b.* -> a.py"),
247 | ],
248 | ids=lambda x: x if isinstance(x, str) else "", # Use description as test ID
249 | )
250 | def test_search_files_include_exclude(
251 | self, file_paths, pattern, paths_include_glob, paths_exclude_glob, expected_matched_files, description
252 | ):
253 | """
254 | Test the include/exclude glob filtering logic in search_files using PathSpec patterns.
255 | """
256 | results = search_files(
257 | relative_file_paths=file_paths,
258 | pattern=pattern,
259 | file_reader=mock_reader_always_match,
260 | paths_include_glob=paths_include_glob,
261 | paths_exclude_glob=paths_exclude_glob,
262 | context_lines_before=0, # No context needed for this test focus
263 | context_lines_after=0,
264 | )
265 |
266 | # Extract the source file paths from the results
267 | actual_matched_files = sorted([result.source_file_path for result in results if result.source_file_path])
268 |
269 | # Assert that the matched files are exactly the ones expected
270 | assert actual_matched_files == sorted(expected_matched_files)
271 |
272 | # Basic check on results structure if files were expected
273 | if expected_matched_files:
274 | assert len(results) == len(expected_matched_files)
275 | for result in results:
276 | assert len(result.matched_lines) == 1 # Mock reader returns one matching line
277 | assert result.matched_lines[0].line_content == "This line contains a match."
278 | assert result.matched_lines[0].match_type == LineType.MATCH
279 |
280 | @pytest.mark.parametrize(
281 | "file_paths, pattern, paths_include_glob, paths_exclude_glob, expected_matched_files, description",
282 | [
283 | # Glob patterns that were problematic with gitignore syntax
284 | (
285 | ["src/serena/agent.py", "src/serena/process_isolated_agent.py", "test/agent.py"],
286 | "match",
287 | "src/**agent.py",
288 | None,
289 | ["src/serena/agent.py", "src/serena/process_isolated_agent.py"],
290 | "Glob: src/**agent.py should match files ending with agent.py under src/",
291 | ),
292 | (
293 | ["src/serena/agent.py", "src/serena/process_isolated_agent.py", "other/agent.py"],
294 | "match",
295 | "**agent.py",
296 | None,
297 | ["src/serena/agent.py", "src/serena/process_isolated_agent.py", "other/agent.py"],
298 | "Glob: **agent.py should match files ending with agent.py anywhere",
299 | ),
300 | (
301 | ["dir/subdir/file.py", "dir/other/file.py", "elsewhere/file.py"],
302 | "match",
303 | "dir/**file.py",
304 | None,
305 | ["dir/subdir/file.py", "dir/other/file.py"],
306 | "Glob: dir/**file.py should match files ending with file.py under dir/",
307 | ),
308 | (
309 | ["src/a/b/c/test.py", "src/x/test.py", "other/test.py"],
310 | "match",
311 | "src/**/test.py",
312 | None,
313 | ["src/a/b/c/test.py", "src/x/test.py"],
314 | "Glob: src/**/test.py should match test.py files under src/ at any depth",
315 | ),
316 | # Edge cases for ** patterns
317 | (
318 | ["agent.py", "src/agent.py", "src/serena/agent.py"],
319 | "match",
320 | "**agent.py",
321 | None,
322 | ["agent.py", "src/agent.py", "src/serena/agent.py"],
323 | "Glob: **agent.py should match at root and any depth",
324 | ),
325 | (["file.txt", "src/file.txt"], "match", "src/**", None, ["src/file.txt"], "Glob: src/** should match everything under src/"),
326 | ],
327 | ids=lambda x: x if isinstance(x, str) else "", # Use description as test ID
328 | )
329 | def test_search_files_glob_patterns(
330 | self, file_paths, pattern, paths_include_glob, paths_exclude_glob, expected_matched_files, description
331 | ):
332 | """
333 | Test glob patterns that were problematic with the previous gitignore-based implementation.
334 | """
335 | results = search_files(
336 | relative_file_paths=file_paths,
337 | pattern=pattern,
338 | file_reader=mock_reader_always_match,
339 | paths_include_glob=paths_include_glob,
340 | paths_exclude_glob=paths_exclude_glob,
341 | context_lines_before=0,
342 | context_lines_after=0,
343 | )
344 |
345 | # Extract the source file paths from the results
346 | actual_matched_files = sorted([result.source_file_path for result in results if result.source_file_path])
347 |
348 | # Assert that the matched files are exactly the ones expected
349 | assert actual_matched_files == sorted(
350 | expected_matched_files
351 | ), f"Pattern '{paths_include_glob}' failed: expected {sorted(expected_matched_files)}, got {actual_matched_files}"
352 |
353 | # Basic check on results structure if files were expected
354 | if expected_matched_files:
355 | assert len(results) == len(expected_matched_files)
356 | for result in results:
357 | assert len(result.matched_lines) == 1 # Mock reader returns one matching line
358 | assert result.matched_lines[0].line_content == "This line contains a match."
359 | assert result.matched_lines[0].match_type == LineType.MATCH
360 |
361 | @pytest.mark.parametrize(
362 | "file_paths, pattern, paths_include_glob, paths_exclude_glob, expected_matched_files, description",
363 | [
364 | # Brace expansion in include glob
365 | (
366 | ["a.py", "b.js", "c.txt"],
367 | "match",
368 | "*.{py,js}",
369 | None,
370 | ["a.py", "b.js"],
371 | "Brace expansion in include glob",
372 | ),
373 | # Brace expansion in exclude glob
374 | (
375 | ["a.py", "b.log", "c.txt"],
376 | "match",
377 | "*.{py,log,txt}",
378 | "*.{log,txt}",
379 | ["a.py"],
380 | "Brace expansion in exclude glob",
381 | ),
382 | # Brace expansion in both include and exclude
383 | (
384 | ["src/a.ts", "src/b.js", "test/a.ts", "test/b.js"],
385 | "match",
386 | "**/*.{ts,js}",
387 | "test/**/*.{ts,js}",
388 | ["src/a.ts", "src/b.js"],
389 | "Brace expansion in both include and exclude",
390 | ),
391 | # No matching files with brace expansion
392 | (
393 | ["a.py", "b.js"],
394 | "match",
395 | "*.{c,h}",
396 | None,
397 | [],
398 | "Brace expansion with no matching files",
399 | ),
400 | # Multiple brace expansions
401 | (
402 | ["src/a/a.py", "src/b/b.py", "lib/a/a.py", "lib/b/b.py"],
403 | "match",
404 | "{src,lib}/{a,b}/*.py",
405 | "lib/b/*.py",
406 | ["src/a/a.py", "src/b/b.py", "lib/a/a.py"],
407 | "Multiple brace expansions in include/exclude",
408 | ),
409 | ],
410 | ids=lambda x: x if isinstance(x, str) else "",
411 | )
412 | def test_search_files_with_brace_expansion(
413 | self, file_paths, pattern, paths_include_glob, paths_exclude_glob, expected_matched_files, description
414 | ):
415 | """Test search_files with glob patterns containing brace expansions."""
416 | results = search_files(
417 | relative_file_paths=file_paths,
418 | pattern=pattern,
419 | file_reader=mock_reader_always_match,
420 | paths_include_glob=paths_include_glob,
421 | paths_exclude_glob=paths_exclude_glob,
422 | )
423 |
424 | actual_matched_files = sorted([result.source_file_path for result in results if result.source_file_path])
425 | assert actual_matched_files == sorted(expected_matched_files), f"Test failed: {description}"
426 |
427 | def test_search_files_no_pattern_match_in_content(self):
428 | """Test that no results are returned if the pattern doesn't match the file content, even if files pass filters."""
429 | file_paths = ["a.py", "b.txt"]
430 | pattern = "non_existent_pattern_in_mock_content" # This won't match mock_reader_always_match content
431 | results = search_files(
432 | relative_file_paths=file_paths,
433 | pattern=pattern,
434 | file_reader=mock_reader_always_match, # Content is "This line contains a match."
435 | paths_include_glob=None, # Both files would pass filters
436 | paths_exclude_glob=None,
437 | )
438 | assert len(results) == 0, "Should not find matches if pattern doesn't match content"
439 |
440 | def test_search_files_regex_pattern_with_filters(self):
441 | """Test using a regex pattern works correctly along with include/exclude filters."""
442 |
443 | def specific_mock_reader(file_path: str) -> str:
444 | # Provide different content for different files to test regex matching
445 | if file_path == "a.py": # noqa: SIM116
446 | return "File A: value=123\nFile A: value=456"
447 | elif file_path == "b.py":
448 | return "File B: value=789"
449 | elif file_path == "c.txt":
450 | return "File C: value=000"
451 | return "No values here."
452 |
453 | file_paths = ["a.py", "b.py", "c.txt"]
454 | pattern = r"value=(\d+)"
455 |
456 | results = search_files(
457 | relative_file_paths=file_paths,
458 | pattern=pattern,
459 | file_reader=specific_mock_reader,
460 | paths_include_glob="*.py", # Only include .py files
461 | paths_exclude_glob="b.*", # Exclude files starting with b
462 | )
463 |
464 | # Expected: a.py included, b.py excluded by glob, c.txt excluded by glob
465 | # a.py has two matches for the regex pattern
466 | assert len(results) == 2, "Expected 2 matches only from a.py"
467 | actual_matched_files = sorted([result.source_file_path for result in results if result.source_file_path])
468 | assert actual_matched_files == ["a.py", "a.py"], "Both matches should be from a.py"
469 | # Check the content of the matched lines
470 | assert results[0].matched_lines[0].line_content == "File A: value=123"
471 | assert results[1].matched_lines[0].line_content == "File A: value=456"
472 |
473 | def test_search_files_context_lines_with_filters(self):
474 | """Test context lines are included correctly when filters are active."""
475 |
476 | def context_mock_reader(file_path: str) -> str:
477 | if file_path == "include_me.txt":
478 | return "Line before 1\nLine before 2\nMATCH HERE\nLine after 1\nLine after 2"
479 | elif file_path == "exclude_me.log":
480 | return "Noise\nMATCH HERE\nNoise"
481 | return "No match"
482 |
483 | file_paths = ["include_me.txt", "exclude_me.log"]
484 | pattern = "MATCH HERE"
485 |
486 | results = search_files(
487 | relative_file_paths=file_paths,
488 | pattern=pattern,
489 | file_reader=context_mock_reader,
490 | paths_include_glob="*.txt", # Only include .txt files
491 | paths_exclude_glob=None,
492 | context_lines_before=1,
493 | context_lines_after=1,
494 | )
495 |
496 | # Expected: Only include_me.txt should be processed and matched
497 | assert len(results) == 1, "Expected only one result from the included file"
498 | result = results[0]
499 | assert result.source_file_path == "include_me.txt"
500 | assert len(result.lines) == 3, "Expected 3 lines (1 before, 1 match, 1 after)"
501 | assert result.lines[0].line_content == "Line before 2", "Incorrect 'before' context line"
502 | assert result.lines[0].match_type == LineType.BEFORE_MATCH
503 | assert result.lines[1].line_content == "MATCH HERE", "Incorrect 'match' line"
504 | assert result.lines[1].match_type == LineType.MATCH
505 | assert result.lines[2].line_content == "Line after 1", "Incorrect 'after' context line"
506 | assert result.lines[2].match_type == LineType.AFTER_MATCH
507 |
508 |
509 | class TestGlobMatch:
510 | """Test the glob_match function directly."""
511 |
512 | @pytest.mark.parametrize(
513 | "pattern, path, expected",
514 | [
515 | # Basic wildcard patterns
516 | ("*.py", "file.py", True),
517 | ("*.py", "file.txt", False),
518 | ("*agent.py", "agent.py", True),
519 | ("*agent.py", "process_isolated_agent.py", True),
520 | ("*agent.py", "agent_test.py", False),
521 | # Double asterisk patterns
522 | ("**agent.py", "agent.py", True),
523 | ("**agent.py", "src/agent.py", True),
524 | ("**agent.py", "src/serena/agent.py", True),
525 | ("**agent.py", "src/serena/process_isolated_agent.py", True),
526 | ("**agent.py", "agent_test.py", False),
527 | # Prefix with double asterisk
528 | ("src/**agent.py", "src/agent.py", True),
529 | ("src/**agent.py", "src/serena/agent.py", True),
530 | ("src/**agent.py", "src/serena/process_isolated_agent.py", True),
531 | ("src/**agent.py", "other/agent.py", False),
532 | ("src/**agent.py", "src/agent_test.py", False),
533 | # Directory patterns
534 | ("src/**", "src/file.py", True),
535 | ("src/**", "src/dir/file.py", True),
536 | ("src/**", "other/file.py", False),
537 | # Exact matches with double asterisk
538 | ("src/**/test.py", "src/test.py", True),
539 | ("src/**/test.py", "src/a/b/test.py", True),
540 | ("src/**/test.py", "src/test_file.py", False),
541 | # Simple patterns without asterisks
542 | ("src/file.py", "src/file.py", True),
543 | ("src/file.py", "src/other.py", False),
544 | ],
545 | )
546 | def test_glob_match(self, pattern, path, expected):
547 | """Test glob_match function with various patterns."""
548 | from src.serena.text_utils import glob_match
549 |
550 | assert glob_match(pattern, path) == expected
551 |
552 |
553 | class TestExpandBraces:
554 | """Test the expand_braces function."""
555 |
556 | @pytest.mark.parametrize(
557 | "pattern, expected",
558 | [
559 | # Basic case
560 | ("src/*.{js,ts}", ["src/*.js", "src/*.ts"]),
561 | # No braces
562 | ("src/*.py", ["src/*.py"]),
563 | # Multiple brace sets
564 | ("src/{a,b}/{c,d}.py", ["src/a/c.py", "src/a/d.py", "src/b/c.py", "src/b/d.py"]),
565 | # Empty string
566 | ("", [""]),
567 | # Braces with empty elements
568 | ("src/{a,,b}.py", ["src/a.py", "src/.py", "src/b.py"]),
569 | # No commas
570 | ("src/{a}.py", ["src/a.py"]),
571 | ],
572 | )
573 | def test_expand_braces(self, pattern, expected):
574 | """Test brace expansion for glob patterns."""
575 | from serena.text_utils import expand_braces
576 |
577 | assert sorted(expand_braces(pattern)) == sorted(expected)
578 |
```