This is page 3 of 8. Use http://codebase.md/moisnx/arc?page={x} to view the full context.
# Directory Structure
```
├── .clang-format
├── .config
│ └── arceditor
│ ├── config.yaml
│ ├── keybinds.conf
│ └── themes
│ ├── catppuccin-mocha.theme
│ ├── cyberpunk-neon.theme
│ ├── default.theme
│ ├── dracula.theme
│ ├── github_dark.theme
│ ├── gruvbox_dark.theme
│ ├── gruvbox_light.theme
│ ├── high_constrast_dark.theme
│ ├── monokai.theme
│ ├── onedark.theme
│ ├── solarized_dark.theme
│ ├── solarized_light.theme
│ ├── tokyo_night.theme
│ └── vscode_light.theme
├── .github
│ └── assets
│ └── screenshot.gif
├── .gitignore
├── .gitmessage
├── .gitmodules
├── build.md
├── CMakeLists.txt
├── deps
│ └── tree-sitter-markdown
│ ├── .editorconfig
│ ├── .gitattributes
│ ├── .github
│ │ ├── screenshot.png
│ │ └── workflows
│ │ ├── ci.yml
│ │ ├── publish.yml
│ │ └── release.yml
│ ├── .gitignore
│ ├── binding.gyp
│ ├── bindings
│ │ ├── go
│ │ │ ├── binding_test.go
│ │ │ ├── markdown_inline.go
│ │ │ └── markdown.go
│ │ ├── node
│ │ │ ├── binding_test.js
│ │ │ ├── binding.cc
│ │ │ ├── index.d.ts
│ │ │ ├── index.js
│ │ │ └── inline.js
│ │ ├── python
│ │ │ ├── tests
│ │ │ │ └── test_binding.py
│ │ │ └── tree_sitter_markdown
│ │ │ ├── __init__.py
│ │ │ ├── __init__.pyi
│ │ │ ├── binding.c
│ │ │ └── py.typed
│ │ ├── rust
│ │ │ ├── benchmark.rs
│ │ │ ├── build.rs
│ │ │ ├── lib.rs
│ │ │ └── parser.rs
│ │ └── swift
│ │ ├── .gitignore
│ │ └── TreeSitterMarkdownTests
│ │ └── TreeSitterMarkdownTests.swift
│ ├── Cargo.toml
│ ├── CMakeLists.txt
│ ├── common
│ │ ├── common.js
│ │ ├── common.mak
│ │ └── html_entities.json
│ ├── CONTRIBUTING.md
│ ├── go.mod
│ ├── LICENSE
│ ├── Makefile
│ ├── package-lock.json
│ ├── package.json
│ ├── Package.resolved
│ ├── Package.swift
│ ├── pyproject.toml
│ ├── README.md
│ ├── scripts
│ │ ├── build.js
│ │ └── test.js
│ ├── setup.py
│ ├── tree-sitter-markdown
│ │ ├── bindings
│ │ │ ├── c
│ │ │ │ ├── tree-sitter-markdown.h
│ │ │ │ └── tree-sitter-markdown.pc.in
│ │ │ └── swift
│ │ │ └── TreeSitterMarkdown
│ │ │ └── markdown.h
│ │ ├── CMakeLists.txt
│ │ ├── grammar.js
│ │ ├── Makefile
│ │ ├── package.json
│ │ ├── queries
│ │ │ ├── highlights.scm
│ │ │ └── injections.scm
│ │ ├── src
│ │ │ ├── grammar.json
│ │ │ ├── node-types.json
│ │ │ ├── parser.c
│ │ │ ├── scanner.c
│ │ │ └── tree_sitter
│ │ │ ├── alloc.h
│ │ │ ├── array.h
│ │ │ └── parser.h
│ │ └── test
│ │ └── corpus
│ │ ├── extension_minus_metadata.txt
│ │ ├── extension_pipe_table.txt
│ │ ├── extension_plus_metadata.txt
│ │ ├── extension_task_list.txt
│ │ ├── failing.txt
│ │ ├── issues.txt
│ │ └── spec.txt
│ ├── tree-sitter-markdown-inline
│ │ ├── bindings
│ │ │ ├── c
│ │ │ │ ├── tree-sitter-markdown-inline.h
│ │ │ │ └── tree-sitter-markdown-inline.pc.in
│ │ │ └── swift
│ │ │ └── TreeSitterMarkdownInline
│ │ │ └── markdown_inline.h
│ │ ├── CMakeLists.txt
│ │ ├── grammar.js
│ │ ├── Makefile
│ │ ├── package.json
│ │ ├── queries
│ │ │ ├── highlights.scm
│ │ │ └── injections.scm
│ │ ├── src
│ │ │ ├── grammar.json
│ │ │ ├── node-types.json
│ │ │ ├── parser.c
│ │ │ ├── scanner.c
│ │ │ └── tree_sitter
│ │ │ ├── alloc.h
│ │ │ ├── array.h
│ │ │ └── parser.h
│ │ └── test
│ │ └── corpus
│ │ ├── extension_latex.txt
│ │ ├── extension_strikethrough.txt
│ │ ├── extension_wikilink.txt
│ │ ├── failing.txt
│ │ ├── issues.txt
│ │ ├── spec.txt
│ │ └── tags.txt
│ └── tree-sitter.json
├── LICENSE
├── Makefile
├── quickstart.md
├── README.md
├── src
│ ├── core
│ │ ├── buffer.cpp
│ │ ├── buffer.h
│ │ ├── config_manager.cpp
│ │ ├── config_manager.h
│ │ ├── editor_delta.h
│ │ ├── editor_validation.h
│ │ ├── editor.cpp
│ │ └── editor.h
│ ├── features
│ │ ├── markdown_state.h
│ │ ├── syntax_config_loader.cpp
│ │ ├── syntax_config_loader.h
│ │ ├── syntax_highlighter.cpp
│ │ └── syntax_highlighter.h
│ ├── main.cpp
│ └── ui
│ ├── input_handler.cpp
│ ├── input_handler.h
│ ├── renderer.cpp
│ ├── renderer.h
│ ├── style_manager.cpp
│ └── style_manager.h
└── treesitter
├── languages.yaml
└── queries
├── _javascript
│ ├── highlights.scm
│ ├── locals.scm
│ └── tags.scm
├── _jsx
│ ├── highlights.scm
│ ├── indents.scm
│ └── textobjects.scm
├── _typescript
│ ├── highlights.scm
│ ├── indents.scm
│ ├── locals.scm
│ ├── tags.scm
│ └── textobjects.scm
├── bash
│ ├── highlights.scm
│ ├── indents.scm
│ ├── injections.scm
│ ├── rainbows.scm
│ ├── tags.scm
│ └── textobjects.scm
├── c
│ ├── highlights.scm
│ ├── indents.scm
│ ├── injections.scm
│ ├── locals.scm
│ ├── rainbows.scm
│ ├── tags.scm
│ └── textobjects.scm
├── cpp
│ ├── highlights.scm
│ ├── indents.scm
│ ├── injections.scm
│ ├── rainbows.scm
│ ├── tags.scm
│ └── textobjects.scm
├── css
│ ├── highlights.scm
│ ├── indents.scm
│ ├── injections.scm
│ └── rainbows.scm
├── ecma
│ ├── highlights.scm
│ ├── indents.scm
│ ├── injections.scm
│ ├── locals.scm
│ ├── rainbows.scm
│ ├── README.md
│ └── textobjects.scm
├── go
│ ├── highlights.scm
│ ├── indents.scm
│ ├── injections.scm
│ ├── locals.scm
│ ├── rainbows.scm
│ ├── tags.scm
│ └── textobjects.scm
├── javascript
│ ├── highlights.scm
│ ├── indents.scm
│ ├── injections.scm
│ ├── locals.scm
│ ├── rainbows.scm
│ ├── tags.scm
│ └── textobjects.scm
├── markdown
│ ├── highlights.scm
│ ├── injections.scm
│ └── tags.scm
├── markdown.inline
│ ├── highlights.scm
│ └── injections.scm
├── python
│ ├── highlights.scm
│ ├── indents.scm
│ ├── injections.scm
│ ├── locals.scm
│ ├── rainbows.scm
│ ├── tags.scm
│ └── textobjects.scm
├── rust
│ ├── highlights.scm
│ ├── indents.scm
│ ├── injections.scm
│ ├── locals.scm
│ ├── rainbows.scm
│ ├── tags.scm
│ └── textobjects.scm
├── toml
│ ├── highlights.scm
│ ├── injections.scm
│ ├── rainbows.scm
│ └── textobjects.scm
├── tsx
│ ├── highlights.scm
│ ├── indents.scm
│ ├── injections.scm
│ ├── locals.scm
│ ├── rainbows.scm
│ ├── tags.scm
│ └── textobjects.scm
├── typescript
│ ├── highlights.scm
│ ├── indents.scm
│ ├── injections.scm
│ ├── locals.scm
│ ├── rainbows.scm
│ ├── tags.scm
│ └── textobjects.scm
├── yaml
│ ├── highlights.scm
│ ├── indents.scm
│ ├── injections.scm
│ ├── rainbows.scm
│ └── textobjects.scm
└── zig
├── highlights.scm
├── indents.scm
├── injections.scm
└── textobjects.scm
```
# Files
--------------------------------------------------------------------------------
/deps/tree-sitter-markdown/tree-sitter-markdown/grammar.js:
--------------------------------------------------------------------------------
```javascript
// This grammar only concerns the block structure according to the CommonMark Spec
// (https://spec.commonmark.org/0.30/#blocks-and-inlines)
// For more information see README.md
/// <reference types="tree-sitter-cli/dsl" />
const common = require('../common/common');
const PRECEDENCE_LEVEL_LINK = common.PRECEDENCE_LEVEL_LINK;
const PUNCTUATION_CHARACTERS_REGEX = '!-/:-@\\[-`\\{-~';
module.exports = grammar({
name: 'markdown',
rules: {
document: $ => seq(
optional(choice(
common.EXTENSION_MINUS_METADATA ? $.minus_metadata : choice(),
common.EXTENSION_PLUS_METADATA ? $.plus_metadata : choice(),
)),
alias(prec.right(repeat($._block_not_section)), $.section),
repeat($.section),
),
...common.rules,
_last_token_punctuation: $ => choice(), // needed for compatability with common rules
// BLOCK STRUCTURE
// All blocks. Every block contains a trailing newline.
_block: $ => choice(
$._block_not_section,
$.section,
),
_block_not_section: $ => choice(
alias($._setext_heading1, $.setext_heading),
alias($._setext_heading2, $.setext_heading),
$.paragraph,
$.indented_code_block,
$.block_quote,
$.thematic_break,
$.list,
$.fenced_code_block,
$._blank_line,
$.html_block,
$.link_reference_definition,
common.EXTENSION_PIPE_TABLE ? $.pipe_table : choice(),
),
section: $ => choice($._section1, $._section2, $._section3, $._section4, $._section5, $._section6),
_section1: $ => prec.right(seq(
alias($._atx_heading1, $.atx_heading),
repeat(choice(
alias(choice($._section6, $._section5, $._section4, $._section3, $._section2), $.section),
$._block_not_section
))
)),
_section2: $ => prec.right(seq(
alias($._atx_heading2, $.atx_heading),
repeat(choice(
alias(choice($._section6, $._section5, $._section4, $._section3), $.section),
$._block_not_section
))
)),
_section3: $ => prec.right(seq(
alias($._atx_heading3, $.atx_heading),
repeat(choice(
alias(choice($._section6, $._section5, $._section4), $.section),
$._block_not_section
))
)),
_section4: $ => prec.right(seq(
alias($._atx_heading4, $.atx_heading),
repeat(choice(
alias(choice($._section6, $._section5), $.section),
$._block_not_section
))
)),
_section5: $ => prec.right(seq(
alias($._atx_heading5, $.atx_heading),
repeat(choice(
alias($._section6, $.section),
$._block_not_section
))
)),
_section6: $ => prec.right(seq(
alias($._atx_heading6, $.atx_heading),
repeat($._block_not_section)
)),
// LEAF BLOCKS
// A thematic break. This is currently handled by the external scanner but maybe could be
// parsed using normal tree-sitter rules.
//
// https://github.github.com/gfm/#thematic-breaks
thematic_break: $ => seq($._thematic_break, choice($._newline, $._eof)),
// An ATX heading. This is currently handled by the external scanner but maybe could be
// parsed using normal tree-sitter rules.
//
// https://github.github.com/gfm/#atx-headings
_atx_heading1: $ => prec(1, seq(
$.atx_h1_marker,
optional($._atx_heading_content),
$._newline
)),
_atx_heading2: $ => prec(1, seq(
$.atx_h2_marker,
optional($._atx_heading_content),
$._newline
)),
_atx_heading3: $ => prec(1, seq(
$.atx_h3_marker,
optional($._atx_heading_content),
$._newline
)),
_atx_heading4: $ => prec(1, seq(
$.atx_h4_marker,
optional($._atx_heading_content),
$._newline
)),
_atx_heading5: $ => prec(1, seq(
$.atx_h5_marker,
optional($._atx_heading_content),
$._newline
)),
_atx_heading6: $ => prec(1, seq(
$.atx_h6_marker,
optional($._atx_heading_content),
$._newline
)),
_atx_heading_content: $ => prec(1, seq(
optional($._whitespace),
field('heading_content', alias($._line, $.inline))
)),
// A setext heading. The underlines are currently handled by the external scanner but maybe
// could be parsed using normal tree-sitter rules.
//
// https://github.github.com/gfm/#setext-headings
_setext_heading1: $ => seq(
field('heading_content', $.paragraph),
$.setext_h1_underline,
choice($._newline, $._eof),
),
_setext_heading2: $ => seq(
field('heading_content', $.paragraph),
$.setext_h2_underline,
choice($._newline, $._eof),
),
// An indented code block. An indented code block is made up of indented chunks and blank
// lines. The indented chunks are handeled by the external scanner.
//
// https://github.github.com/gfm/#indented-code-blocks
indented_code_block: $ => prec.right(seq($._indented_chunk, repeat(choice($._indented_chunk, $._blank_line)))),
_indented_chunk: $ => seq($._indented_chunk_start, repeat(choice($._line, $._newline)), $._block_close, optional($.block_continuation)),
// A fenced code block. Fenced code blocks are mainly handled by the external scanner. In
// case of backtick code blocks the external scanner also checks that the info string is
// proper.
//
// https://github.github.com/gfm/#fenced-code-blocks
fenced_code_block: $ => prec.right(choice(
seq(
alias($._fenced_code_block_start_backtick, $.fenced_code_block_delimiter),
optional($._whitespace),
optional($.info_string),
$._newline,
optional($.code_fence_content),
optional(seq(alias($._fenced_code_block_end_backtick, $.fenced_code_block_delimiter), $._close_block, $._newline)),
$._block_close,
),
seq(
alias($._fenced_code_block_start_tilde, $.fenced_code_block_delimiter),
optional($._whitespace),
optional($.info_string),
$._newline,
optional($.code_fence_content),
optional(seq(alias($._fenced_code_block_end_tilde, $.fenced_code_block_delimiter), $._close_block, $._newline)),
$._block_close,
),
)),
code_fence_content: $ => repeat1(choice($._newline, $._line)),
info_string: $ => choice(
seq($.language, repeat(choice($._line, $.backslash_escape, $.entity_reference, $.numeric_character_reference))),
seq(
repeat1(choice('{', '}')),
optional(choice(
seq($.language, repeat(choice($._line, $.backslash_escape, $.entity_reference, $.numeric_character_reference))),
seq($._whitespace, repeat(choice($._line, $.backslash_escape, $.entity_reference, $.numeric_character_reference))),
))
)
),
language: $ => prec.right(repeat1(choice($._word, common.punctuation_without($, ['{', '}', ',']), $.backslash_escape, $.entity_reference, $.numeric_character_reference))),
// An HTML block. We do not emit addition nodes relating to the kind or structure or of the
// html block as this is best done using language injections and a proper html parsers.
//
// See the `build_html_block` function for more information.
// See the spec for the different kinds of html blocks.
//
// https://github.github.com/gfm/#html-blocks
html_block: $ => prec(1, seq(optional($._whitespace), choice(
$._html_block_1,
$._html_block_2,
$._html_block_3,
$._html_block_4,
$._html_block_5,
$._html_block_6,
$._html_block_7,
))),
_html_block_1: $ => build_html_block($,
// new RegExp(
// '[ \t]*<' + regex_case_insensitive_list(HTML_TAG_NAMES_RULE_1) + '([\\r\\n]|[ \\t>][^<\\r\\n]*(\\n|\\r\\n?)?)'
// ),
$._html_block_1_start,
$._html_block_1_end,
true
),
_html_block_2: $ => build_html_block($, $._html_block_2_start, '-->', true),
_html_block_3: $ => build_html_block($, $._html_block_3_start, '?>', true),
_html_block_4: $ => build_html_block($, $._html_block_4_start, '>', true),
_html_block_5: $ => build_html_block($, $._html_block_5_start, ']]>', true),
_html_block_6: $ => build_html_block(
$,
$._html_block_6_start,
seq($._newline, $._blank_line),
true
),
_html_block_7: $ => build_html_block(
$,
$._html_block_7_start,
seq($._newline, $._blank_line),
false
),
// A link reference definition. We need to make sure that this is not mistaken for a
// paragraph or indented chunk. The `$._no_indented_chunk` token is used to tell the
// external scanner not to allow indented chunks when the `$.link_title` of the link
// reference definition would be valid.
//
// https://github.github.com/gfm/#link-reference-definitions
link_reference_definition: $ => prec.dynamic(PRECEDENCE_LEVEL_LINK, seq(
optional($._whitespace),
$.link_label,
':',
optional(seq(optional($._whitespace), optional(seq($._soft_line_break, optional($._whitespace))))),
$.link_destination,
optional(prec.dynamic(2 * PRECEDENCE_LEVEL_LINK, seq(
choice(
seq($._whitespace, optional(seq($._soft_line_break, optional($._whitespace)))),
seq($._soft_line_break, optional($._whitespace)),
),
optional($._no_indented_chunk),
$.link_title
))),
choice($._newline, $._soft_line_break, $._eof),
)),
_text_inline_no_link: $ => choice($._word, $._whitespace, common.punctuation_without($, ['[', ']'])),
// A paragraph. The parsing tactic for deciding when a paragraph ends is as follows:
// on every newline inside a paragraph a conflict is triggered manually using
// `$._split_token` to split the parse state into two branches.
//
// One of them - the one that also contains a `$._soft_line_break_marker` will try to
// continue the paragraph, but we make sure that the beginning of a new block that can
// interrupt a paragraph can also be parsed. If this is the case we know that the paragraph
// should have been closed and the external parser will emit an `$._error` to kill the parse
// branch.
//
// The other parse branch consideres the paragraph to be over. It will be killed if no valid new
// block is detected before the next newline. (For example it will also be killed if a indented
// code block is detected, which cannot interrupt paragraphs).
//
// Either way, after the next newline only one branch will exist, so the ammount of branches
// related to paragraphs ending does not grow.
//
// https://github.github.com/gfm/#paragraphs
paragraph: $ => seq(alias(repeat1(choice($._line, $._soft_line_break)), $.inline), choice($._newline, $._eof)),
// A blank line including the following newline.
//
// https://github.github.com/gfm/#blank-lines
_blank_line: $ => seq($._blank_line_start, choice($._newline, $._eof)),
// CONTAINER BLOCKS
// A block quote. This is the most basic example of a container block handled by the
// external scanner.
//
// https://github.github.com/gfm/#block-quotes
block_quote: $ => seq(
alias($._block_quote_start, $.block_quote_marker),
optional($.block_continuation),
repeat($._block),
$._block_close,
optional($.block_continuation)
),
// A list. This grammar does not differentiate between loose and tight lists for efficiency
// reasons.
//
// Lists can only contain list items with list markers of the same type. List items are
// handled by the external scanner.
//
// https://github.github.com/gfm/#lists
list: $ => prec.right(choice(
$._list_plus,
$._list_minus,
$._list_star,
$._list_dot,
$._list_parenthesis
)),
_list_plus: $ => prec.right(repeat1(alias($._list_item_plus, $.list_item))),
_list_minus: $ => prec.right(repeat1(alias($._list_item_minus, $.list_item))),
_list_star: $ => prec.right(repeat1(alias($._list_item_star, $.list_item))),
_list_dot: $ => prec.right(repeat1(alias($._list_item_dot, $.list_item))),
_list_parenthesis: $ => prec.right(repeat1(alias($._list_item_parenthesis, $.list_item))),
// Some list items can not interrupt a paragraph and are marked as such by the external
// scanner.
list_marker_plus: $ => choice($._list_marker_plus, $._list_marker_plus_dont_interrupt),
list_marker_minus: $ => choice($._list_marker_minus, $._list_marker_minus_dont_interrupt),
list_marker_star: $ => choice($._list_marker_star, $._list_marker_star_dont_interrupt),
list_marker_dot: $ => choice($._list_marker_dot, $._list_marker_dot_dont_interrupt),
list_marker_parenthesis: $ => choice($._list_marker_parenthesis, $._list_marker_parenthesis_dont_interrupt),
_list_item_plus: $ => seq(
$.list_marker_plus,
optional($.block_continuation),
$._list_item_content,
$._block_close,
optional($.block_continuation)
),
_list_item_minus: $ => seq(
$.list_marker_minus,
optional($.block_continuation),
$._list_item_content,
$._block_close,
optional($.block_continuation)
),
_list_item_star: $ => seq(
$.list_marker_star,
optional($.block_continuation),
$._list_item_content,
$._block_close,
optional($.block_continuation)
),
_list_item_dot: $ => seq(
$.list_marker_dot,
optional($.block_continuation),
$._list_item_content,
$._block_close,
optional($.block_continuation)
),
_list_item_parenthesis: $ => seq(
$.list_marker_parenthesis,
optional($.block_continuation),
$._list_item_content,
$._block_close,
optional($.block_continuation)
),
// List items are closed after two consecutive blank lines
_list_item_content: $ => choice(
prec(1, seq(
$._blank_line,
$._blank_line,
$._close_block,
optional($.block_continuation)
)),
repeat1($._block),
common.EXTENSION_TASK_LIST ? prec(1, seq(
choice($.task_list_marker_checked, $.task_list_marker_unchecked),
$._whitespace,
$.paragraph,
repeat($._block)
)) : choice()
),
// Newlines as in the spec. Parsing a newline triggers the matching process by making
// the external parser emit a `$._line_ending`.
_newline: $ => seq(
$._line_ending,
optional($.block_continuation)
),
_soft_line_break: $ => seq(
$._soft_line_ending,
optional($.block_continuation)
),
// Some symbols get parsed as single tokens so that html blocks get detected properly
_line: $ => prec.right(repeat1(choice($._word, $._whitespace, common.punctuation_without($, [])))),
_word: $ => choice(
new RegExp('[^' + PUNCTUATION_CHARACTERS_REGEX + ' \\t\\n\\r]+'),
common.EXTENSION_TASK_LIST ? choice(
/\[[xX]\]/,
/\[[ \t]\]/,
) : choice()
),
// The external scanner emits some characters that should just be ignored.
_whitespace: $ => /[ \t]+/,
...(common.EXTENSION_TASK_LIST ? {
task_list_marker_checked: $ => prec(1, /\[[xX]\]/),
task_list_marker_unchecked: $ => prec(1, /\[[ \t]\]/),
} : {}),
...(common.EXTENSION_PIPE_TABLE ? {
pipe_table: $ => prec.right(seq(
$._pipe_table_start,
alias($.pipe_table_row, $.pipe_table_header),
$._newline,
$.pipe_table_delimiter_row,
repeat(seq($._pipe_table_newline, optional($.pipe_table_row))),
choice($._newline, $._eof),
)),
_pipe_table_newline: $ => seq(
$._pipe_table_line_ending,
optional($.block_continuation)
),
pipe_table_delimiter_row: $ => seq(
optional(seq(
optional($._whitespace),
'|',
)),
repeat1(prec.right(seq(
optional($._whitespace),
$.pipe_table_delimiter_cell,
optional($._whitespace),
'|',
))),
optional($._whitespace),
optional(seq(
$.pipe_table_delimiter_cell,
optional($._whitespace)
)),
),
pipe_table_delimiter_cell: $ => seq(
optional(alias(':', $.pipe_table_align_left)),
repeat1('-'),
optional(alias(':', $.pipe_table_align_right)),
),
pipe_table_row: $ => seq(
optional(seq(
optional($._whitespace),
'|',
)),
choice(
seq(
repeat1(prec.right(seq(
choice(
seq(
optional($._whitespace),
$.pipe_table_cell,
optional($._whitespace)
),
alias($._whitespace, $.pipe_table_cell)
),
'|',
))),
optional($._whitespace),
optional(seq(
$.pipe_table_cell,
optional($._whitespace)
)),
),
seq(
optional($._whitespace),
$.pipe_table_cell,
optional($._whitespace)
)
),
),
pipe_table_cell: $ => prec.right(seq(
choice(
$._word,
$._backslash_escape,
common.punctuation_without($, ['|']),
),
repeat(choice(
$._word,
$._whitespace,
$._backslash_escape,
common.punctuation_without($, ['|']),
)),
)),
} : {}),
},
externals: $ => [
// Quite a few of these tokens could maybe be implemented without use of the external parser.
// For this the `$._open_block` and `$._close_block` tokens should be used to tell the external
// parser to put a new anonymous block on the block stack.
// Block structure gets parsed as follows: After every newline (`$._line_ending`) we try to match
// as many open blocks as possible. For example if the last line was part of a block quote we look
// for a `>` at the beginning of the next line. We emit a `$.block_continuation` for each matched
// block. For this process the external scanner keeps a stack of currently open blocks.
//
// If we are not able to match all blocks that does not necessarily mean that all unmatched blocks
// have to be closed. It could also mean that the line is a lazy continuation line
// (https://github.github.com/gfm/#lazy-continuation-line, see also `$._split_token` and
// `$._soft_line_break_marker` below)
//
// If a block does get closed (because it was not matched or because some closing token was
// encountered) we emit a `$._block_close` token
$._line_ending, // this token does not contain the actual newline characters. see `$._newline`
$._soft_line_ending,
$._block_close,
$.block_continuation,
// Tokens signifying the start of a block. Blocks that do not need a `$._block_close` because they
// always span one line are marked as such.
$._block_quote_start,
$._indented_chunk_start,
$.atx_h1_marker, // atx headings do not need a `$._block_close`
$.atx_h2_marker,
$.atx_h3_marker,
$.atx_h4_marker,
$.atx_h5_marker,
$.atx_h6_marker,
$.setext_h1_underline, // setext headings do not need a `$._block_close`
$.setext_h2_underline,
$._thematic_break, // thematic breaks do not need a `$._block_close`
$._list_marker_minus,
$._list_marker_plus,
$._list_marker_star,
$._list_marker_parenthesis,
$._list_marker_dot,
$._list_marker_minus_dont_interrupt, // list items that do not interrupt an ongoing paragraph
$._list_marker_plus_dont_interrupt,
$._list_marker_star_dont_interrupt,
$._list_marker_parenthesis_dont_interrupt,
$._list_marker_dot_dont_interrupt,
$._fenced_code_block_start_backtick,
$._fenced_code_block_start_tilde,
$._blank_line_start, // Does not contain the newline characters. Blank lines do not need a `$._block_close`
// Special tokens for block structure
// Closing backticks or tildas for a fenced code block. They are used to trigger a `$._close_block`
// which in turn will trigger a `$._block_close` at the beginning the following line.
$._fenced_code_block_end_backtick,
$._fenced_code_block_end_tilde,
$._html_block_1_start,
$._html_block_1_end,
$._html_block_2_start,
$._html_block_3_start,
$._html_block_4_start,
$._html_block_5_start,
$._html_block_6_start,
$._html_block_7_start,
// Similarly this is used if the closing of a block is not decided by the external parser.
// A `$._block_close` will be emitted at the beginning of the next line. Notice that a
// `$._block_close` can also get emitted if the parent block closes.
$._close_block,
// This is a workaround so the external parser does not try to open indented blocks when
// parsing a link reference definition.
$._no_indented_chunk,
// An `$._error` token is never valid and gets emmited to kill invalid parse branches. Concretely
// this is used to decide wether a newline closes a paragraph and together and it gets emitted
// when trying to parse the `$._trigger_error` token in `$.link_title`.
$._error,
$._trigger_error,
$._eof,
$.minus_metadata,
$.plus_metadata,
$._pipe_table_start,
$._pipe_table_line_ending,
],
precedences: $ => [
[$._setext_heading1, $._block],
[$._setext_heading2, $._block],
[$.indented_code_block, $._block],
],
conflicts: $ => [
[$.link_reference_definition],
[$.link_label, $._line],
[$.link_reference_definition, $._line],
],
extras: $ => [],
});
// General purpose structure for html blocks. The different kinds mostly work the same but have
// different openling and closing conditions. Some html blocks may not interrupt a paragraph and
// have to be marked as such.
function build_html_block($, open, close, interrupt_paragraph) {
return seq(
open,
repeat(choice(
$._line,
$._newline,
seq(close, $._close_block),
)),
$._block_close,
optional($.block_continuation),
);
}
```
--------------------------------------------------------------------------------
/src/features/syntax_highlighter.cpp:
--------------------------------------------------------------------------------
```cpp
#include "syntax_highlighter.h"
#include "src/core/config_manager.h"
#include <algorithm>
#include <cstring>
#include <fstream>
#include <sstream>
#ifdef _WIN32
#include <curses.h>
#else
#include <ncurses.h>
#endif
#include <iostream>
#ifdef TREE_SITTER_ENABLED
#include "language_registry.h" // Auto-generated by CMake
#include "tree_sitter/api.h"
#endif
SyntaxHighlighter::SyntaxHighlighter()
: config_loader_(std::make_unique<SyntaxConfigLoader>()),
current_language_config_(nullptr), currentLanguage("text")
#ifdef TREE_SITTER_ENABLED
,
parser_(nullptr), tree_(nullptr), current_ts_language_(nullptr),
current_ts_query_(nullptr)
#endif
{
#ifdef TREE_SITTER_ENABLED
initializeTreeSitter();
#endif
}
SyntaxHighlighter::~SyntaxHighlighter()
{
#ifdef TREE_SITTER_ENABLED
cleanupTreeSitter();
#endif
}
bool SyntaxHighlighter::initialize(const std::string &config_directory)
{
// std::cerr << "=== SyntaxHighlighter::initialize ===\n";
// std::cerr << "Config directory: " << config_directory << std::endl;
if (!config_loader_->loadAllLanguageConfigs(config_directory))
{
std::cerr << "Failed to load language configurations from: "
<< config_directory << std::endl;
// Fall back to basic highlighting rules
loadBasicRules();
return false;
}
ConfigManager::registerReloadCallback(
[this, config_directory]()
{
std::cerr << "Syntax config reload triggered." << std::endl;
// Clear old configs and reload them
config_loader_->language_configs_.clear();
config_loader_->extension_to_language_.clear();
// Reload all config files from the directory
this->config_loader_->loadAllLanguageConfigs(
ConfigManager::getSyntaxRulesDir());
// Re-apply the parser for the current file
setLanguage(this->currentLanguage); // Re-set language to pick up new
// rules/queries
// NOTE: Force a full buffer re-highlight/re-parse (e.g., set a flag)
});
// std::cout << "Successfully loaded language configurations" << std::endl;
return true;
}
#ifdef TREE_SITTER_ENABLED
void SyntaxHighlighter::diagnoseGrammar() const
{
if (!current_ts_language_)
{
std::cerr << "ERROR: No language loaded" << std::endl;
return;
}
std::cerr << "=== Grammar Diagnostic ===" << std::endl;
std::cerr << "ABI Version: " << ts_language_abi_version(current_ts_language_)
<< std::endl;
std::cerr << "Symbol count: "
<< ts_language_symbol_count(current_ts_language_) << std::endl;
// Test a simple parse
const char *test_code = "int x;";
TSTree *test_tree = ts_parser_parse_string(parser_, nullptr, test_code,
std::strlen(test_code));
if (test_tree)
{
TSNode root = ts_tree_root_node(test_tree);
char *tree_string = ts_node_string(root);
std::cerr << "Parse test result: " << tree_string << std::endl;
free(tree_string);
ts_tree_delete(test_tree);
}
else
{
std::cerr << "ERROR: Failed to parse simple test code" << std::endl;
}
std::cerr << "=== End Diagnostic ===" << std::endl;
}
#endif
void SyntaxHighlighter::setLanguage(const std::string &extension)
{
std::string language_name =
config_loader_->getLanguageFromExtension(extension);
const LanguageConfig *config =
config_loader_->getLanguageConfig(language_name);
if (config)
{
current_language_config_ = config;
currentLanguage = language_name;
#ifdef TREE_SITTER_ENABLED
if (!config->parser_name.empty() && parser_)
{
const TSLanguage *ts_language = getLanguageFunction(config->parser_name);
if (ts_language)
{
if (!ts_parser_set_language(parser_, ts_language))
{
std::cerr << "ERROR: Failed to set language for parser" << std::endl;
loadBasicRules();
return;
}
current_ts_language_ = ts_language;
// Clean up old query
if (current_ts_query_)
{
ts_query_delete(current_ts_query_);
current_ts_query_ = nullptr;
}
// Load and merge all queries
if (!config->queries.empty())
{
std::string merged_query_source;
for (const auto &query_path : config->queries)
{
std::ifstream file(query_path);
if (!file.is_open())
{
std::cerr << "ERROR: Cannot open query file: " << query_path
<< std::endl;
continue;
}
std::stringstream buffer;
buffer << file.rdbuf();
std::string query_content = buffer.str();
if (!query_content.empty())
{
// Add newline between queries for safety
if (!merged_query_source.empty())
{
merged_query_source += "\n";
}
merged_query_source += query_content;
}
}
// Parse the merged query once
if (!merged_query_source.empty())
{
uint32_t error_offset;
TSQueryError error_type;
current_ts_query_ = ts_query_new(
current_ts_language_, merged_query_source.c_str(),
merged_query_source.length(), &error_offset, &error_type);
if (!current_ts_query_)
{
std::cerr << "ERROR: Failed to parse merged query" << std::endl;
std::cerr << " Error offset: " << error_offset << std::endl;
std::cerr << " Error type: " << error_type << std::endl;
// Show context around error
if (error_offset < merged_query_source.length())
{
int context_start = std::max(0, (int)error_offset - 50);
int context_end = std::min((int)merged_query_source.length(),
(int)error_offset + 50);
std::cerr << "Context around error:" << std::endl;
std::cerr << "..."
<< merged_query_source.substr(
context_start, context_end - context_start)
<< "..." << std::endl;
std::cerr << std::string(error_offset - context_start + 3, ' ')
<< "^" << std::endl;
}
}
}
}
}
else
{
std::cerr << "ERROR: No Tree-sitter language function found for: "
<< config->parser_name << std::endl;
loadBasicRules();
}
}
else
{
std::cerr << "Tree-sitter not available or no parser specified, using "
"basic highlighting"
<< std::endl;
loadBasicRules();
}
#else
std::cerr << "Tree-sitter disabled, using basic highlighting" << std::endl;
loadBasicRules();
#endif
}
else
{
std::cerr << "ERROR: No config found for language: " << language_name
<< std::endl;
loadBasicRules();
currentLanguage = "text";
current_language_config_ = nullptr;
}
}
std::vector<ColorSpan>
SyntaxHighlighter::getHighlightSpans(const std::string &line, int lineIndex,
const GapBuffer &buffer) const
{
// Check cache first
auto cache_it = line_cache_.find(lineIndex);
if (cache_it != line_cache_.end())
{
return cache_it->second;
}
// Handle Markdown special states
if (currentLanguage == "Markdown" && line_states_.count(lineIndex))
{
MarkdownState state = line_states_.at(lineIndex);
if (state == MarkdownState::IN_FENCED_CODE_BLOCK)
{
std::vector<ColorSpan> result = {
{0, (int)line.length(), getColorPairValue("MARKDOWN_CODE_BLOCK"),
A_NORMAL, 100}};
line_cache_[lineIndex] = result;
return result;
}
else if (state == MarkdownState::IN_BLOCKQUOTE)
{
std::vector<ColorSpan> result = {
{0, (int)line.length(), getColorPairValue("MARKDOWN_BLOCKQUOTE"),
A_NORMAL, 90}};
line_cache_[lineIndex] = result;
return result;
}
}
std::vector<ColorSpan> result;
#ifdef TREE_SITTER_ENABLED
// CRITICAL: Do lazy reparse if needed
if (tree_needs_reparse_)
{
const_cast<SyntaxHighlighter *>(this)->updateTree(buffer);
const_cast<SyntaxHighlighter *>(this)->tree_needs_reparse_ = false;
}
if (current_ts_query_ && tree_)
{
try
{
result = executeTreeSitterQuery(line, lineIndex);
}
catch (const std::exception &e)
{
std::cerr << "Tree-sitter query error on line " << lineIndex << ": "
<< e.what() << std::endl;
result = getBasicHighlightSpans(line);
}
}
#endif
// Fall back to basic highlighting if no Tree-sitter result
if (result.empty())
{
result = getBasicHighlightSpans(line);
}
// Cache the result
line_cache_[lineIndex] = result;
return result;
}
void SyntaxHighlighter::updateTreeAfterEdit(
const GapBuffer &buffer, size_t byte_pos, size_t old_byte_len,
size_t new_byte_len, uint32_t start_row, uint32_t start_col,
uint32_t old_end_row, uint32_t old_end_col, uint32_t new_end_row,
uint32_t new_end_col)
{
#ifdef TREE_SITTER_ENABLED
if (!tree_ || !parser_)
return;
// Apply incremental edit to tree structure
TSInputEdit edit = {.start_byte = (uint32_t)byte_pos,
.old_end_byte = (uint32_t)(byte_pos + old_byte_len),
.new_end_byte = (uint32_t)(byte_pos + new_byte_len),
.start_point = {start_row, start_col},
.old_end_point = {old_end_row, old_end_col},
.new_end_point = {new_end_row, new_end_col}};
ts_tree_edit(tree_, &edit);
tree_version_++;
// Mark that tree needs reparsing (will happen on next query)
tree_needs_reparse_ = true;
// For very large changes, schedule background reparse
if (old_end_row != new_end_row && (new_end_row - old_end_row) > 10)
{
scheduleBackgroundParse(buffer);
}
#endif
}
void SyntaxHighlighter::invalidateLineCache(int lineNum)
{
line_cache_.erase(lineNum);
}
void SyntaxHighlighter::bufferChanged(const GapBuffer &buffer)
{
#ifdef TREE_SITTER_ENABLED
if (!parser_ || !current_ts_language_)
return;
// REMOVED the "optimization" that was skipping reparsing
// If current_buffer_content_ is empty, we MUST reparse
if (current_buffer_content_.empty())
{
// Content was cleared - this signals we need full reparse
updateTree(buffer);
}
else if (!tree_)
{
// No tree exists - need initial parse
updateTree(buffer);
}
// If tree exists AND content is valid, incremental edits should have
// already updated it via notifyEdit()
#endif
if (currentLanguage == "Markdown")
{
updateMarkdownState(buffer);
}
}
void SyntaxHighlighter::invalidateFromLine(int startLine)
{
// This is for structural changes (insert/delete lines)
// Clear only lines >= startLine, but do it efficiently
auto it = line_cache_.lower_bound(startLine);
if (it != line_cache_.end())
{
line_cache_.erase(it, line_cache_.end());
}
// Don't clear content cache unless change is massive
// Let incremental edits handle the tree updates
}
#ifdef TREE_SITTER_ENABLED
bool SyntaxHighlighter::initializeTreeSitter()
{
parser_ = ts_parser_new();
if (!parser_)
{
std::cerr << "ERROR: Failed to create Tree-sitter parser" << std::endl;
return false;
}
// Auto-register all languages from generated header
registerAllLanguages(language_registry_);
// std::cerr << "Tree-sitter initialized with " << language_registry_.size()
// << " language parser(s)" << std::endl;
return true;
}
void SyntaxHighlighter::cleanupTreeSitter()
{
// Wait for background thread
while (is_parsing_)
{
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
std::lock_guard<std::mutex> lock(tree_mutex_); // ADD LOCK
if (current_ts_query_)
{
ts_query_delete(current_ts_query_);
current_ts_query_ = nullptr;
}
if (tree_)
{
ts_tree_delete(tree_);
tree_ = nullptr;
}
if (parser_)
{
ts_parser_delete(parser_);
parser_ = nullptr;
}
}
const TSLanguage *
SyntaxHighlighter::getLanguageFunction(const std::string &parser_name)
{
auto it = language_registry_.find(parser_name);
if (it != language_registry_.end())
{
return it->second(); // Call the function pointer
}
// Enhanced error message showing available languages
std::cerr << "WARNING: No Tree-sitter language found for: '" << parser_name
<< "'" << std::endl;
std::cerr << " Available languages: ";
bool first = true;
for (const auto &pair : language_registry_)
{
if (!first)
std::cerr << ", ";
std::cerr << pair.first;
first = false;
}
std::cerr << std::endl;
return nullptr;
}
TSQuery *
SyntaxHighlighter::loadQueryFromFile(const std::string &query_file_path)
{
std::ifstream file(query_file_path);
if (!file.is_open())
{
std::cerr << "ERROR: Cannot open query file: " << query_file_path
<< std::endl;
return nullptr;
}
std::stringstream buffer;
buffer << file.rdbuf();
std::string query_source = buffer.str();
if (query_source.empty())
{
std::cerr << "ERROR: Query file is empty: " << query_file_path << std::endl;
return nullptr;
}
// Debug: Print the query source around the error offset
// std::cerr << "Query source length: " << query_source.length() << "
// characters"
// << std::endl;
uint32_t error_offset;
TSQueryError error_type;
TSQuery *query =
ts_query_new(current_ts_language_, query_source.c_str(),
query_source.length(), &error_offset, &error_type);
if (!query)
{
std::cerr << "ERROR: Failed to parse query file " << query_file_path
<< std::endl;
std::cerr << " Error offset: " << error_offset << std::endl;
std::cerr << " Error type: " << error_type;
// Provide more detailed error information
switch (error_type)
{
case TSQueryErrorNone:
std::cerr << " (None)";
break;
case TSQueryErrorSyntax:
std::cerr << " (Syntax Error)";
break;
case TSQueryErrorNodeType:
std::cerr << " (Unknown Node Type)";
break;
case TSQueryErrorField:
std::cerr << " (Unknown Field)";
break;
case TSQueryErrorCapture:
std::cerr << " (Unknown Capture)";
break;
case TSQueryErrorStructure:
std::cerr << " (Invalid Structure)";
break;
default:
std::cerr << " (Unknown Error)";
break;
}
std::cerr << std::endl;
// Show context around error
if (error_offset < query_source.length())
{
int context_start = std::max(0, (int)error_offset - 50);
int context_end =
std::min((int)query_source.length(), (int)error_offset + 50);
std::cerr << "Context around error:" << std::endl;
std::cerr << "..."
<< query_source.substr(context_start,
context_end - context_start)
<< "..." << std::endl;
// Point to error location
std::cerr << std::string(error_offset - context_start + 3, ' ') << "^"
<< std::endl;
}
return nullptr;
}
// std::cerr << "Successfully loaded query from: " << query_file_path
// << std::endl;
return query;
}
void SyntaxHighlighter::notifyEdit(size_t byte_pos, size_t old_byte_len,
size_t new_byte_len, uint32_t start_row,
uint32_t start_col, uint32_t old_end_row,
uint32_t old_end_col, uint32_t new_end_row,
uint32_t new_end_col)
{
#ifdef TREE_SITTER_ENABLED
if (!tree_)
{
return;
}
TSInputEdit edit = {.start_byte = (uint32_t)byte_pos,
.old_end_byte = (uint32_t)(byte_pos + old_byte_len),
.new_end_byte = (uint32_t)(byte_pos + new_byte_len),
.start_point = {start_row, start_col},
.old_end_point = {old_end_row, old_end_col},
.new_end_point = {new_end_row, new_end_col}};
ts_tree_edit(tree_, &edit);
// CRITICAL FIX: Mark that we need to reparse on next access
// This forces updateTree() to be called on next getHighlightSpans()
// current_buffer_content_.clear();
#endif
}
void SyntaxHighlighter::invalidateLineRange(int startLine, int endLine)
{
// OPTIMIZATION: Only invalidate affected lines, not entire cache
// For single-line changes, only clear that line
if (endLine - startLine <= 3)
{
for (int i = startLine; i <= endLine; ++i)
{
line_cache_.erase(i);
line_states_.erase(i);
}
return;
}
// For multi-line changes, clear from startLine onwards
auto cache_it = line_cache_.lower_bound(startLine);
if (cache_it != line_cache_.end())
{
line_cache_.erase(cache_it, line_cache_.end());
}
auto state_it = line_states_.lower_bound(startLine);
if (state_it != line_states_.end())
{
line_states_.erase(state_it, line_states_.end());
}
// DON'T clear buffer content unless structural change
if (endLine - startLine > 10)
{
current_buffer_content_.clear(); // Force reparse on next access
}
}
void SyntaxHighlighter::updateTree(const GapBuffer &buffer)
{
#ifdef TREE_SITTER_ENABLED
std::string content;
int lineCount = buffer.getLineCount();
// Build line offset cache while building content
line_byte_offsets_.clear();
line_byte_offsets_.reserve(lineCount + 1);
line_byte_offsets_.push_back(0); // First line starts at byte 0
for (int i = 0; i < lineCount; i++)
{
if (i > 0)
content += "\n";
content += buffer.getLine(i);
// Store the byte offset for the next line
line_byte_offsets_.push_back(content.length());
}
if (content.empty())
{
std::cerr << "WARNING: Attempting to parse empty buffer\n";
return;
}
std::lock_guard<std::mutex> lock(tree_mutex_);
current_buffer_content_ = content;
if (!tree_)
{
tree_ = ts_parser_parse_string(parser_, nullptr, content.c_str(),
content.length());
}
else
{
TSTree *old_tree = tree_;
tree_ = ts_parser_parse_string(parser_, old_tree, content.c_str(),
content.length());
if (old_tree && tree_)
{
ts_tree_delete(old_tree);
}
}
if (!tree_)
{
std::cerr << "ERROR: Failed to parse tree\n";
}
#endif
}
void SyntaxHighlighter::markViewportLines(int startLine, int endLine) const
{
priority_lines_.clear();
for (int i = startLine; i <= endLine; ++i)
{
priority_lines_.insert(i);
}
}
bool SyntaxHighlighter::isLineHighlighted(int lineIndex) const
{
return line_cache_.find(lineIndex) != line_cache_.end();
}
std::vector<ColorSpan>
SyntaxHighlighter::executeTreeSitterQuery(const std::string &line,
int lineNum) const
{
if (!current_ts_query_ || !tree_)
return {};
std::lock_guard<std::mutex> lock(tree_mutex_);
std::vector<ColorSpan> spans;
TSQueryCursor *cursor = ts_query_cursor_new();
TSNode root_node = ts_tree_root_node(tree_);
int adjusted_line =
is_full_parse_ ? lineNum : (lineNum - viewport_start_line_);
if (adjusted_line < 0 ||
adjusted_line >= ts_node_end_point(root_node).row + 1)
{
ts_query_cursor_delete(cursor);
return {};
}
// Calculate byte range for current line
uint32_t line_start_byte = 0;
uint32_t line_end_byte = 0;
std::istringstream content_stream(current_buffer_content_);
std::string content_line;
int current_line = 0;
while (std::getline(content_stream, content_line) && current_line <= lineNum)
{
if (current_line == lineNum)
{
line_end_byte = line_start_byte + content_line.length();
break;
}
line_start_byte += content_line.length() + 1;
current_line++;
}
ts_query_cursor_set_byte_range(cursor, line_start_byte, line_end_byte);
ts_query_cursor_exec(cursor, current_ts_query_, root_node);
TSQueryMatch match;
while (ts_query_cursor_next_match(cursor, &match))
{
for (uint32_t i = 0; i < match.capture_count; i++)
{
TSQueryCapture capture = match.captures[i];
TSNode node = capture.node;
TSPoint start_point = ts_node_start_point(node);
TSPoint end_point = ts_node_end_point(node);
// ORIGINAL: Only process captures starting on current line
// Check if this capture affects the current line
if (start_point.row <= (uint32_t)lineNum &&
end_point.row >= (uint32_t)lineNum)
{
uint32_t name_length;
const char *capture_name_ptr = ts_query_capture_name_for_id(
current_ts_query_, capture.index, &name_length);
std::string capture_name(capture_name_ptr, name_length);
int start_col =
(start_point.row == (uint32_t)lineNum) ? start_point.column : 0;
int end_col = (end_point.row == (uint32_t)lineNum) ? end_point.column
: (int)line.length();
start_col = std::max(0, std::min(start_col, (int)line.length()));
end_col = std::max(start_col, std::min(end_col, (int)line.length()));
if (start_col < end_col)
{
int color_pair = getColorPairForCapture(capture_name);
spans.push_back({start_col, end_col, color_pair, 0, 100});
}
}
}
}
ts_query_cursor_delete(cursor);
return spans;
}
int SyntaxHighlighter::getColorPairForCapture(
const std::string &capture_name) const
{
static const std::unordered_map<std::string, std::string> capture_to_color = {
// Keywords
{"keyword", "KEYWORD"},
{"keyword.control", "KEYWORD"},
{"keyword.function", "KEYWORD"},
{"keyword.operator", "KEYWORD"},
{"keyword.return", "KEYWORD"},
{"keyword.conditional", "KEYWORD"},
{"keyword.repeat", "KEYWORD"},
{"keyword.import", "KEYWORD"},
{"keyword.exception", "KEYWORD"},
// Types
{"type", "TYPE"},
{"type.builtin", "TYPE"},
{"type.definition", "TYPE"},
{"class", "TYPE"},
{"interface", "TYPE"},
// Functions
{"function", "FUNCTION"},
{"function.call", "FUNCTION"},
{"function.builtin", "FUNCTION"},
{"function.method", "FUNCTION"},
{"method", "FUNCTION"},
// Variables & constants
{"variable", "VARIABLE"},
{"variable.parameter", "VARIABLE"},
{"variable.builtin", "CONSTANT"},
{"variable.member", "VARIABLE"},
{"constant", "CONSTANT"},
{"constant.builtin", "CONSTANT"},
{"parameter", "VARIABLE"},
// Literals
{"string", "STRING_LITERAL"},
{"string_literal", "STRING_LITERAL"},
{"number", "NUMBER"},
{"integer", "NUMBER"},
{"float", "NUMBER"},
{"boolean", "CONSTANT"},
// Comments
{"comment", "COMMENT"},
// Operators & punctuation
{"operator", "OPERATOR"},
{"punctuation", "PUNCTUATION"},
{"punctuation.bracket", "PUNCTUATION"},
{"punctuation.delimiter", "PUNCTUATION"},
// Specialized
{"namespace", "NAMESPACE"},
{"property", "PROPERTY"},
{"field", "PROPERTY"},
{"attribute", "DECORATOR"},
{"decorator", "DECORATOR"},
{"label", "LABEL"},
{"tag", "LABEL"},
// Preprocessor/macro
{"preproc", "MACRO"},
{"preproc_include", "MACRO"},
{"preproc_def", "MACRO"},
{"preproc_call", "MACRO"},
{"preproc_if", "MACRO"},
{"preproc_ifdef", "MACRO"},
{"preproc_ifndef", "MACRO"},
{"preproc_else", "MACRO"},
{"preproc_elif", "MACRO"},
{"preproc_endif", "MACRO"},
{"macro", "MACRO"},
// Markup (Markdown, etc.)
{"markup.heading", "MARKUP_HEADING"},
{"heading", "MARKUP_HEADING"},
{"markup.bold", "MARKUP_BOLD"},
{"markup.italic", "MARKUP_ITALIC"},
{"emphasis", "MARKUP_ITALIC"},
{"markup.code", "MARKUP_CODE"},
{"code", "MARKUP_CODE"},
{"markup.link", "MARKUP_LINK"},
{"link_text", "MARKUP_LINK"},
{"markup.url", "MARKUP_URL"},
{"link_uri", "MARKUP_URL"},
{"markup.quote", "MARKUP_BLOCKQUOTE"},
{"markup.list", "MARKUP_LIST"},
{"markup.code", "MARKUP_CODE"},
{"code_fence_content", "MARKUP_CODE_BLOCK"},
{"code_span", "MARKUP_CODE"},
// Markdown structure
{"markup.list", "MARKUP_LIST"},
{"markup.quote", "MARKUP_BLOCKQUOTE"},
};
auto it = capture_to_color.find(capture_name);
if (it != capture_to_color.end())
{
return getColorPairValue(it->second);
}
// Fallback: hierarchical matching
if (capture_name.find("keyword") != std::string::npos)
return getColorPairValue("KEYWORD");
if (capture_name.find("type") != std::string::npos)
return getColorPairValue("TYPE");
if (capture_name.find("function") != std::string::npos)
return getColorPairValue("FUNCTION");
if (capture_name.find("string") != std::string::npos)
return getColorPairValue("STRING_LITERAL");
if (capture_name.find("comment") != std::string::npos)
return getColorPairValue("COMMENT");
if (capture_name.find("number") != std::string::npos)
return getColorPairValue("NUMBER");
if (capture_name.find("constant") != std::string::npos)
return getColorPairValue("CONSTANT");
return 0; // Default
}
#endif
int SyntaxHighlighter::getColorPairValue(const std::string &color_name) const
{
static const std::unordered_map<std::string, int> color_map = {
{"COMMENT", COMMENT},
{"KEYWORD", KEYWORD},
{"STRING_LITERAL", STRING_LITERAL},
{"NUMBER", NUMBER},
{"FUNCTION", FUNCTION},
{"VARIABLE", VARIABLE},
{"TYPE", TYPE},
{"OPERATOR", OPERATOR},
{"PUNCTUATION", PUNCTUATION},
{"CONSTANT", CONSTANT},
{"NAMESPACE", NAMESPACE},
{"PROPERTY", PROPERTY},
{"DECORATOR", DECORATOR},
{"MACRO", MACRO},
{"LABEL", LABEL},
{"MARKUP_HEADING", MARKUP_HEADING},
{"MARKUP_BOLD", MARKUP_BOLD},
{"MARKUP_ITALIC", MARKUP_ITALIC},
{"MARKUP_CODE", MARKUP_CODE},
{"MARKUP_CODE_BLOCK", MARKUP_CODE_BLOCK},
{"MARKUP_LINK", MARKUP_LINK},
{"MARKUP_URL", MARKUP_URL},
{"MARKUP_LIST", MARKUP_LIST},
{"MARKUP_BLOCKQUOTE", MARKUP_BLOCKQUOTE},
{"MARKUP_STRIKETHROUGH", MARKUP_STRIKETHROUGH},
{"MARKUP_QUOTE", MARKUP_QUOTE}};
auto it = color_map.find(color_name);
return (it != color_map.end()) ? it->second : 0;
}
int SyntaxHighlighter::getAttributeValue(
const std::string &attribute_name) const
{
static const std::unordered_map<std::string, int> attribute_map = {
{"0", 0},
{"A_BOLD", A_BOLD},
{"A_DIM", A_DIM},
{"A_UNDERLINE", A_UNDERLINE},
{"A_REVERSE", A_REVERSE}};
auto it = attribute_map.find(attribute_name);
return (it != attribute_map.end()) ? it->second : 0;
}
std::vector<ColorSpan>
SyntaxHighlighter::getBasicHighlightSpans(const std::string &line) const
{
std::vector<ColorSpan> spans;
// Very basic regex-based highlighting as fallback
// Comments (# and //)
size_t comment_pos = line.find('#');
if (comment_pos == std::string::npos)
{
comment_pos = line.find("//");
}
if (comment_pos != std::string::npos)
{
spans.push_back({static_cast<int>(comment_pos),
static_cast<int>(line.length()),
getColorPairValue("COMMENT"), 0, 100});
}
// Simple string detection (basic)
bool in_string = false;
char string_char = 0;
size_t string_start = 0;
for (size_t i = 0; i < line.length(); i++)
{
char c = line[i];
if (!in_string && (c == '"' || c == '\''))
{
in_string = true;
string_char = c;
string_start = i;
}
else if (in_string && c == string_char && (i == 0 || line[i - 1] != '\\'))
{
spans.push_back({static_cast<int>(string_start), static_cast<int>(i + 1),
getColorPairValue("STRING_LITERAL"), 0, 90});
in_string = false;
}
}
return spans;
}
void SyntaxHighlighter::loadBasicRules()
{
// This is called as a fallback when Tree-sitter is not available
std::cerr << "Loading basic highlighting rules (fallback mode)" << std::endl;
}
// Markdown state management (unchanged from original)
void SyntaxHighlighter::updateMarkdownState(const GapBuffer &buffer)
{
if (currentLanguage != "Markdown")
{
line_states_.clear();
return;
}
line_states_.clear();
MarkdownState currentState = MarkdownState::DEFAULT;
int lineCount = buffer.getLineCount();
for (int i = 0; i < lineCount; ++i)
{
std::string line = buffer.getLine(i);
line_states_[i] = currentState;
if (currentState == MarkdownState::DEFAULT)
{
if (line.rfind("```", 0) == 0)
{
currentState = MarkdownState::IN_FENCED_CODE_BLOCK;
}
else if (line.rfind(">", 0) == 0)
{
line_states_[i] = MarkdownState::IN_BLOCKQUOTE;
}
}
else if (currentState == MarkdownState::IN_FENCED_CODE_BLOCK)
{
if (line.rfind("```", 0) == 0)
{
currentState = MarkdownState::DEFAULT;
}
line_states_[i] = MarkdownState::IN_FENCED_CODE_BLOCK;
}
}
}
std::vector<std::string> SyntaxHighlighter::getSupportedExtensions() const
{
return {"cpp", "h", "hpp", "c", "py", "md", "txt"};
}
void SyntaxHighlighter::debugTreeSitterState() const
{
#ifdef TREE_SITTER_ENABLED
std::cerr << "=== Tree-sitter State Debug ===\n";
std::cerr << "Current language: " << currentLanguage << "\n";
std::cerr << "Parser: " << (parser_ ? "EXISTS" : "NULL") << "\n";
std::cerr << "Tree: " << (tree_ ? "EXISTS" : "NULL") << "\n";
std::cerr << "TS Language: " << (current_ts_language_ ? "EXISTS" : "NULL")
<< "\n";
std::cerr << "TS Query: " << (current_ts_query_ ? "EXISTS" : "NULL") << "\n";
std::cerr << "Buffer content length: " << current_buffer_content_.length()
<< "\n";
std::cerr << "Line cache size: " << line_cache_.size() << "\n";
if (tree_)
{
TSNode root = ts_tree_root_node(tree_);
char *tree_str = ts_node_string(root);
std::cerr << "Parse tree (truncated): "
<< std::string(tree_str).substr(0, 200) << "...\n";
free(tree_str);
}
std::cerr << "=== End Debug ===\n";
#else
std::cerr << "Tree-sitter not enabled\n";
#endif
}
void SyntaxHighlighter::parseViewportOnly(const GapBuffer &buffer,
int targetLine)
{
#ifdef TREE_SITTER_ENABLED
if (!parser_ || !current_ts_language_)
return;
int startLine = std::max(0, targetLine - 50);
int endLine = std::min(buffer.getLineCount() - 1, targetLine + 50);
std::string content;
for (int i = startLine; i <= endLine; i++)
{
if (i > startLine)
content += "\n";
content += buffer.getLine(i);
}
if (content.empty())
return;
TSTree *new_tree = ts_parser_parse_string(parser_, nullptr, content.c_str(),
content.length());
if (new_tree)
{
std::lock_guard<std::mutex> lock(tree_mutex_); // LOCK ADDED
if (tree_)
ts_tree_delete(tree_);
tree_ = new_tree;
current_buffer_content_ = content;
viewport_start_line_ = startLine;
is_full_parse_ = false;
}
#endif
}
void SyntaxHighlighter::scheduleBackgroundParse(const GapBuffer &buffer)
{
#ifdef TREE_SITTER_ENABLED
if (is_parsing_ || !parser_ || !current_ts_language_)
return;
auto now = std::chrono::steady_clock::now();
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
now - last_parse_time_)
.count();
if (elapsed < 500)
return;
// Copy content BEFORE starting thread
std::string content;
int lineCount = buffer.getLineCount();
content.reserve(lineCount * 80);
for (int i = 0; i < lineCount; i++)
{
if (i > 0)
content += "\n";
content += buffer.getLine(i);
}
if (content.empty())
return;
is_parsing_ = true;
last_parse_time_ = now;
// NEW: Capture current version
uint64_t expected_version = tree_version_.load();
// Create a COPY of parser state to avoid races
TSParser *temp_parser = ts_parser_new();
if (!ts_parser_set_language(temp_parser, current_ts_language_))
{
ts_parser_delete(temp_parser);
is_parsing_ = false;
return;
}
parse_thread_ = std::thread(
[this, content, temp_parser, expected_version]() mutable
{
TSTree *new_tree = ts_parser_parse_string(
temp_parser, nullptr, content.c_str(), content.length());
if (new_tree)
{
std::lock_guard<std::mutex> lock(tree_mutex_);
// NEW: Only update if no newer edits happened
if (tree_version_.load() == expected_version)
{
TSTree *old_tree = tree_;
tree_ = new_tree;
current_buffer_content_ = std::move(content);
is_full_parse_ = true;
if (old_tree)
ts_tree_delete(old_tree);
}
else
{
// Discard stale parse - user has made newer edits
ts_tree_delete(new_tree);
}
}
ts_parser_delete(temp_parser);
is_parsing_ = false;
parse_complete_ = true;
});
parse_thread_.detach();
#endif
}
void SyntaxHighlighter::forceFullReparse(const GapBuffer &buffer)
{
#ifdef TREE_SITTER_ENABLED
if (!parser_ || !current_ts_language_)
return;
std::lock_guard<std::mutex> lock(tree_mutex_);
// Build fresh content
std::string content;
int lineCount = buffer.getLineCount();
// Pre-allocate to avoid reallocations
size_t estimated_size = lineCount * 50; // Rough estimate
content.reserve(estimated_size);
for (int i = 0; i < lineCount; i++)
{
if (i > 0)
content += "\n";
content += buffer.getLine(i);
}
if (content.empty())
{
std::cerr << "WARNING: Empty buffer in forceFullReparse\n";
return;
}
// OPTIMIZATION: Use the old tree as a reference for faster re-parsing
TSTree *old_tree = tree_;
tree_ = ts_parser_parse_string(parser_, old_tree, content.c_str(),
content.length());
if (tree_)
{
current_buffer_content_ = std::move(content); // Move instead of copy
is_full_parse_ = true;
// Delete old tree AFTER successful parse
if (old_tree)
ts_tree_delete(old_tree);
}
else
{
std::cerr << "ERROR: Reparse failed, keeping old tree\n";
tree_ = old_tree; // Restore old tree
return;
}
#endif
// Clear cache ONLY, don't rebuild markdown state unless necessary
line_cache_.clear();
if (currentLanguage == "Markdown")
{
updateMarkdownState(buffer);
}
}
void SyntaxHighlighter::clearAllCache()
{
// Clear ALL cached line highlighting
line_cache_.clear();
// Clear line states (for Markdown)
line_states_.clear();
// Clear priority lines
priority_lines_.clear();
// CRITICAL: Force tree-sitter content to be marked as stale
current_buffer_content_.clear();
// Mark that we need a full reparse
is_full_parse_ = false;
}
```
--------------------------------------------------------------------------------
/deps/tree-sitter-markdown/tree-sitter-markdown/src/scanner.c:
--------------------------------------------------------------------------------
```cpp
#include "tree_sitter/parser.h"
#include <assert.h>
#include <ctype.h>
#include <string.h>
#include <wchar.h>
#include <wctype.h>
// For explanation of the tokens see grammar.js
typedef enum {
LINE_ENDING,
SOFT_LINE_ENDING,
BLOCK_CLOSE,
BLOCK_CONTINUATION,
BLOCK_QUOTE_START,
INDENTED_CHUNK_START,
ATX_H1_MARKER,
ATX_H2_MARKER,
ATX_H3_MARKER,
ATX_H4_MARKER,
ATX_H5_MARKER,
ATX_H6_MARKER,
SETEXT_H1_UNDERLINE,
SETEXT_H2_UNDERLINE,
THEMATIC_BREAK,
LIST_MARKER_MINUS,
LIST_MARKER_PLUS,
LIST_MARKER_STAR,
LIST_MARKER_PARENTHESIS,
LIST_MARKER_DOT,
LIST_MARKER_MINUS_DONT_INTERRUPT,
LIST_MARKER_PLUS_DONT_INTERRUPT,
LIST_MARKER_STAR_DONT_INTERRUPT,
LIST_MARKER_PARENTHESIS_DONT_INTERRUPT,
LIST_MARKER_DOT_DONT_INTERRUPT,
FENCED_CODE_BLOCK_START_BACKTICK,
FENCED_CODE_BLOCK_START_TILDE,
BLANK_LINE_START,
FENCED_CODE_BLOCK_END_BACKTICK,
FENCED_CODE_BLOCK_END_TILDE,
HTML_BLOCK_1_START,
HTML_BLOCK_1_END,
HTML_BLOCK_2_START,
HTML_BLOCK_3_START,
HTML_BLOCK_4_START,
HTML_BLOCK_5_START,
HTML_BLOCK_6_START,
HTML_BLOCK_7_START,
CLOSE_BLOCK,
NO_INDENTED_CHUNK,
ERROR,
TRIGGER_ERROR,
TOKEN_EOF,
MINUS_METADATA,
PLUS_METADATA,
PIPE_TABLE_START,
PIPE_TABLE_LINE_ENDING,
} TokenType;
// Description of a block on the block stack.
//
// LIST_ITEM is a list item with minimal indentation (content begins at indent
// level 2) while LIST_ITEM_MAX_INDENTATION represents a list item with maximal
// indentation without being considered a indented code block.
//
// ANONYMOUS represents any block that whose close is not handled by the
// external s.
typedef enum {
BLOCK_QUOTE,
INDENTED_CODE_BLOCK,
LIST_ITEM,
LIST_ITEM_1_INDENTATION,
LIST_ITEM_2_INDENTATION,
LIST_ITEM_3_INDENTATION,
LIST_ITEM_4_INDENTATION,
LIST_ITEM_5_INDENTATION,
LIST_ITEM_6_INDENTATION,
LIST_ITEM_7_INDENTATION,
LIST_ITEM_8_INDENTATION,
LIST_ITEM_9_INDENTATION,
LIST_ITEM_10_INDENTATION,
LIST_ITEM_11_INDENTATION,
LIST_ITEM_12_INDENTATION,
LIST_ITEM_13_INDENTATION,
LIST_ITEM_14_INDENTATION,
LIST_ITEM_MAX_INDENTATION,
FENCED_CODE_BLOCK,
ANONYMOUS,
} Block;
// Determines if a character is punctuation as defined by the markdown spec.
static bool is_punctuation(char chr) {
return (chr >= '!' && chr <= '/') || (chr >= ':' && chr <= '@') ||
(chr >= '[' && chr <= '`') || (chr >= '{' && chr <= '~');
}
// Returns the indentation level which lines of a list item should have at
// minimum. Should only be called with blocks for which `is_list_item` returns
// true.
static uint8_t list_item_indentation(Block block) {
return (uint8_t)(block - LIST_ITEM + 2);
}
#define NUM_HTML_TAG_NAMES_RULE_1 3
static const char *const HTML_TAG_NAMES_RULE_1[NUM_HTML_TAG_NAMES_RULE_1] = {
"pre", "script", "style"};
#define NUM_HTML_TAG_NAMES_RULE_7 62
static const char *const HTML_TAG_NAMES_RULE_7[NUM_HTML_TAG_NAMES_RULE_7] = {
"address", "article", "aside", "base", "basefont", "blockquote",
"body", "caption", "center", "col", "colgroup", "dd",
"details", "dialog", "dir", "div", "dl", "dt",
"fieldset", "figcaption", "figure", "footer", "form", "frame",
"frameset", "h1", "h2", "h3", "h4", "h5",
"h6", "head", "header", "hr", "html", "iframe",
"legend", "li", "link", "main", "menu", "menuitem",
"nav", "noframes", "ol", "optgroup", "option", "p",
"param", "section", "source", "summary", "table", "tbody",
"td", "tfoot", "th", "thead", "title", "tr",
"track", "ul"};
// For explanation of the tokens see grammar.js
static const bool paragraph_interrupt_symbols[] = {
false, // LINE_ENDING,
false, // SOFT_LINE_ENDING,
false, // BLOCK_CLOSE,
false, // BLOCK_CONTINUATION,
true, // BLOCK_QUOTE_START,
false, // INDENTED_CHUNK_START,
true, // ATX_H1_MARKER,
true, // ATX_H2_MARKER,
true, // ATX_H3_MARKER,
true, // ATX_H4_MARKER,
true, // ATX_H5_MARKER,
true, // ATX_H6_MARKER,
true, // SETEXT_H1_UNDERLINE,
true, // SETEXT_H2_UNDERLINE,
true, // THEMATIC_BREAK,
true, // LIST_MARKER_MINUS,
true, // LIST_MARKER_PLUS,
true, // LIST_MARKER_STAR,
true, // LIST_MARKER_PARENTHESIS,
true, // LIST_MARKER_DOT,
false, // LIST_MARKER_MINUS_DONT_INTERRUPT,
false, // LIST_MARKER_PLUS_DONT_INTERRUPT,
false, // LIST_MARKER_STAR_DONT_INTERRUPT,
false, // LIST_MARKER_PARENTHESIS_DONT_INTERRUPT,
false, // LIST_MARKER_DOT_DONT_INTERRUPT,
true, // FENCED_CODE_BLOCK_START_BACKTICK,
true, // FENCED_CODE_BLOCK_START_TILDE,
true, // BLANK_LINE_START,
false, // FENCED_CODE_BLOCK_END_BACKTICK,
false, // FENCED_CODE_BLOCK_END_TILDE,
true, // HTML_BLOCK_1_START,
false, // HTML_BLOCK_1_END,
true, // HTML_BLOCK_2_START,
true, // HTML_BLOCK_3_START,
true, // HTML_BLOCK_4_START,
true, // HTML_BLOCK_5_START,
true, // HTML_BLOCK_6_START,
false, // HTML_BLOCK_7_START,
false, // CLOSE_BLOCK,
false, // NO_INDENTED_CHUNK,
false, // ERROR,
false, // TRIGGER_ERROR,
false, // EOF,
false, // MINUS_METADATA,
false, // PLUS_METADATA,
true, // PIPE_TABLE_START,
false, // PIPE_TABLE_LINE_ENDING,
};
// State bitflags used with `Scanner.state`
// Currently matching (at the beginning of a line)
static const uint8_t STATE_MATCHING = 0x1 << 0;
// Last line break was inside a paragraph
static const uint8_t STATE_WAS_SOFT_LINE_BREAK = 0x1 << 1;
// Block should be closed after next line break
static const uint8_t STATE_CLOSE_BLOCK = 0x1 << 4;
static size_t roundup_32(size_t x) {
x--;
x |= x >> 1;
x |= x >> 2;
x |= x >> 4;
x |= x >> 8;
x |= x >> 16;
x++;
return x;
}
typedef struct {
// A stack of open blocks in the current parse state
struct {
size_t size;
size_t capacity;
Block *items;
} open_blocks;
// Parser state flags
uint8_t state;
// Number of blocks that have been matched so far. Only changes during
// matching and is reset after every line ending
uint8_t matched;
// Consumed but "unused" indentation. Sometimes a tab needs to be "split" to
// be used in multiple tokens.
uint8_t indentation;
// The current column. Used to decide how many spaces a tab should equal
uint8_t column;
// The delimiter length of the currently open fenced code block
uint8_t fenced_code_block_delimiter_length;
bool simulate;
} Scanner;
static void push_block(Scanner *s, Block b) {
if (s->open_blocks.size == s->open_blocks.capacity) {
s->open_blocks.capacity =
s->open_blocks.capacity ? s->open_blocks.capacity << 1 : 8;
void *tmp = realloc(s->open_blocks.items,
sizeof(Block) * s->open_blocks.capacity);
assert(tmp != NULL);
s->open_blocks.items = tmp;
}
s->open_blocks.items[s->open_blocks.size++] = b;
}
static inline Block pop_block(Scanner *s) {
return s->open_blocks.items[--s->open_blocks.size];
}
// Write the whole state of a Scanner to a byte buffer
static unsigned serialize(Scanner *s, char *buffer) {
unsigned size = 0;
buffer[size++] = (char)s->state;
buffer[size++] = (char)s->matched;
buffer[size++] = (char)s->indentation;
buffer[size++] = (char)s->column;
buffer[size++] = (char)s->fenced_code_block_delimiter_length;
size_t blocks_count = s->open_blocks.size;
if (blocks_count > 0) {
memcpy(&buffer[size], s->open_blocks.items,
blocks_count * sizeof(Block));
size += blocks_count * sizeof(Block);
}
return size;
}
// Read the whole state of a Scanner from a byte buffer
// `serizalize` and `deserialize` should be fully symmetric.
static void deserialize(Scanner *s, const char *buffer, unsigned length) {
s->open_blocks.size = 0;
s->open_blocks.capacity = 0;
s->state = 0;
s->matched = 0;
s->indentation = 0;
s->column = 0;
s->fenced_code_block_delimiter_length = 0;
if (length > 0) {
size_t size = 0;
s->state = (uint8_t)buffer[size++];
s->matched = (uint8_t)buffer[size++];
s->indentation = (uint8_t)buffer[size++];
s->column = (uint8_t)buffer[size++];
s->fenced_code_block_delimiter_length = (uint8_t)buffer[size++];
size_t blocks_size = length - size;
if (blocks_size > 0) {
size_t blocks_count = blocks_size / sizeof(Block);
// ensure open blocks has enough room
if (s->open_blocks.capacity < blocks_count) {
size_t capacity = roundup_32(blocks_count);
void *tmp = realloc(s->open_blocks.items,
sizeof(Block) * capacity);
assert(tmp != NULL);
s->open_blocks.items = tmp;
s->open_blocks.capacity = capacity;
}
memcpy(s->open_blocks.items, &buffer[size], blocks_size);
s->open_blocks.size = blocks_count;
}
}
}
static void mark_end(Scanner *s, TSLexer *lexer) {
if (!s->simulate) {
lexer->mark_end(lexer);
}
}
// Convenience function to emit the error token. This is done to stop invalid
// parse branches. Specifically:
// 1. When encountering a newline after a line break that ended a paragraph, and
// no new block
// has been opened.
// 2. When encountering a new block after a soft line break.
// 3. When a `$._trigger_error` token is valid, which is used to stop parse
// branches through
// normal tree-sitter grammar rules.
//
// See also the `$._soft_line_break` and `$._paragraph_end_newline` tokens in
// grammar.js
static bool error(TSLexer *lexer) {
lexer->result_symbol = ERROR;
return true;
}
// Advance the lexer one character
// Also keeps track of the current column, counting tabs as spaces with tab stop
// 4 See https://github.github.com/gfm/#tabs
static size_t advance(Scanner *s, TSLexer *lexer) {
size_t size = 1;
if (lexer->lookahead == '\t') {
size = 4 - s->column;
s->column = 0;
} else {
s->column = (s->column + 1) % 4;
}
lexer->advance(lexer, false);
return size;
}
// Try to match the given block, i.e. consume all tokens that belong to the
// block. These are
// 1. indentation for list items and indented code blocks
// 2. '>' for block quotes
// Returns true if the block is matched and false otherwise
static bool match(Scanner *s, TSLexer *lexer, Block block) {
switch (block) {
case INDENTED_CODE_BLOCK:
while (s->indentation < 4) {
if (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
s->indentation += advance(s, lexer);
} else {
break;
}
}
if (s->indentation >= 4 && lexer->lookahead != '\n' &&
lexer->lookahead != '\r') {
s->indentation -= 4;
return true;
}
break;
case LIST_ITEM:
case LIST_ITEM_1_INDENTATION:
case LIST_ITEM_2_INDENTATION:
case LIST_ITEM_3_INDENTATION:
case LIST_ITEM_4_INDENTATION:
case LIST_ITEM_5_INDENTATION:
case LIST_ITEM_6_INDENTATION:
case LIST_ITEM_7_INDENTATION:
case LIST_ITEM_8_INDENTATION:
case LIST_ITEM_9_INDENTATION:
case LIST_ITEM_10_INDENTATION:
case LIST_ITEM_11_INDENTATION:
case LIST_ITEM_12_INDENTATION:
case LIST_ITEM_13_INDENTATION:
case LIST_ITEM_14_INDENTATION:
case LIST_ITEM_MAX_INDENTATION:
while (s->indentation < list_item_indentation(block)) {
if (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
s->indentation += advance(s, lexer);
} else {
break;
}
}
if (s->indentation >= list_item_indentation(block)) {
s->indentation -= list_item_indentation(block);
return true;
}
if (lexer->lookahead == '\n' || lexer->lookahead == '\r') {
s->indentation = 0;
return true;
}
break;
case BLOCK_QUOTE:
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
s->indentation += advance(s, lexer);
}
if (lexer->lookahead == '>') {
advance(s, lexer);
s->indentation = 0;
if (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
s->indentation += advance(s, lexer) - 1;
}
return true;
}
break;
case FENCED_CODE_BLOCK:
case ANONYMOUS:
return true;
}
return false;
}
static bool parse_fenced_code_block(Scanner *s, const char delimiter,
TSLexer *lexer, const bool *valid_symbols) {
// count the number of backticks
uint8_t level = 0;
while (lexer->lookahead == delimiter) {
advance(s, lexer);
level++;
}
mark_end(s, lexer);
// If this is able to close a fenced code block then that is the only valid
// interpretation. It can only close a fenced code block if the number of
// backticks is at least the number of backticks of the opening delimiter.
// Also it cannot be indented more than 3 spaces.
if ((delimiter == '`' ? valid_symbols[FENCED_CODE_BLOCK_END_BACKTICK]
: valid_symbols[FENCED_CODE_BLOCK_END_TILDE]) &&
s->indentation < 4 && level >= s->fenced_code_block_delimiter_length) {
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
advance(s, lexer);
}
if (lexer->lookahead == '\n' || lexer->lookahead == '\r') {
s->fenced_code_block_delimiter_length = 0;
lexer->result_symbol = delimiter == '`'
? FENCED_CODE_BLOCK_END_BACKTICK
: FENCED_CODE_BLOCK_END_TILDE;
return true;
}
}
// If this could be the start of a fenced code block, check if the info
// string contains any backticks.
if ((delimiter == '`' ? valid_symbols[FENCED_CODE_BLOCK_START_BACKTICK]
: valid_symbols[FENCED_CODE_BLOCK_START_TILDE]) &&
level >= 3) {
bool info_string_has_backtick = false;
if (delimiter == '`') {
while (lexer->lookahead != '\n' && lexer->lookahead != '\r' &&
!lexer->eof(lexer)) {
if (lexer->lookahead == '`') {
info_string_has_backtick = true;
break;
}
advance(s, lexer);
}
}
// If it does not then choose to interpret this as the start of a fenced
// code block.
if (!info_string_has_backtick) {
lexer->result_symbol = delimiter == '`'
? FENCED_CODE_BLOCK_START_BACKTICK
: FENCED_CODE_BLOCK_START_TILDE;
if (!s->simulate)
push_block(s, FENCED_CODE_BLOCK);
// Remember the length of the delimiter for later, since we need it
// to decide whether a sequence of backticks can close the block.
s->fenced_code_block_delimiter_length = level;
s->indentation = 0;
return true;
}
}
return false;
}
static bool parse_star(Scanner *s, TSLexer *lexer, const bool *valid_symbols) {
advance(s, lexer);
mark_end(s, lexer);
// Otherwise count the number of stars permitting whitespaces between them.
size_t star_count = 1;
// Also remember how many stars there are before the first whitespace...
// ...and how many spaces follow the first star.
uint8_t extra_indentation = 0;
for (;;) {
if (lexer->lookahead == '*') {
if (star_count == 1 && extra_indentation >= 1 &&
valid_symbols[LIST_MARKER_STAR]) {
// If we get to this point then the token has to be at least
// this long. We need to call `mark_end` here in case we decide
// later that this is a list item.
mark_end(s, lexer);
}
star_count++;
advance(s, lexer);
} else if (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
if (star_count == 1) {
extra_indentation += advance(s, lexer);
} else {
advance(s, lexer);
}
} else {
break;
}
}
bool line_end = lexer->lookahead == '\n' || lexer->lookahead == '\r';
bool dont_interrupt = false;
if (star_count == 1 && line_end) {
extra_indentation = 1;
// line is empty so don't interrupt paragraphs if this is a list marker
dont_interrupt = s->matched == s->open_blocks.size;
}
// If there were at least 3 stars then this could be a thematic break
bool thematic_break = star_count >= 3 && line_end;
// If there was a star and at least one space after that star then this
// could be a list marker.
bool list_marker_star = star_count >= 1 && extra_indentation >= 1;
if (valid_symbols[THEMATIC_BREAK] && thematic_break && s->indentation < 4) {
// If a thematic break is valid then it takes precedence
lexer->result_symbol = THEMATIC_BREAK;
mark_end(s, lexer);
s->indentation = 0;
return true;
}
if ((dont_interrupt ? valid_symbols[LIST_MARKER_STAR_DONT_INTERRUPT]
: valid_symbols[LIST_MARKER_STAR]) &&
list_marker_star) {
// List markers take precedence over emphasis markers
// If star_count > 1 then we already called mark_end at the right point.
// Otherwise the token should go until this point.
if (star_count == 1) {
mark_end(s, lexer);
}
// Not counting one space...
extra_indentation--;
// ... check if the list item begins with an indented code block
if (extra_indentation <= 3) {
// If not then calculate the indentation level of the list item
// content as indentation of list marker + indentation after list
// marker - 1
extra_indentation += s->indentation;
s->indentation = 0;
} else {
// Otherwise the indentation level is just the indentation of the
// list marker. We keep the indentation after the list marker for
// later blocks.
uint8_t temp = s->indentation;
s->indentation = extra_indentation;
extra_indentation = temp;
}
if (!s->simulate)
push_block(s, (Block)(LIST_ITEM + extra_indentation));
lexer->result_symbol =
dont_interrupt ? LIST_MARKER_STAR_DONT_INTERRUPT : LIST_MARKER_STAR;
return true;
}
return false;
}
static bool parse_thematic_break_underscore(Scanner *s, TSLexer *lexer,
const bool *valid_symbols) {
advance(s, lexer);
mark_end(s, lexer);
size_t underscore_count = 1;
for (;;) {
if (lexer->lookahead == '_') {
underscore_count++;
advance(s, lexer);
} else if (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
advance(s, lexer);
} else {
break;
}
}
bool line_end = lexer->lookahead == '\n' || lexer->lookahead == '\r';
if (underscore_count >= 3 && line_end && valid_symbols[THEMATIC_BREAK]) {
lexer->result_symbol = THEMATIC_BREAK;
mark_end(s, lexer);
s->indentation = 0;
return true;
}
return false;
}
static bool parse_block_quote(Scanner *s, TSLexer *lexer,
const bool *valid_symbols) {
if (valid_symbols[BLOCK_QUOTE_START]) {
advance(s, lexer);
s->indentation = 0;
if (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
s->indentation += advance(s, lexer) - 1;
}
lexer->result_symbol = BLOCK_QUOTE_START;
if (!s->simulate)
push_block(s, BLOCK_QUOTE);
return true;
}
return false;
}
static bool parse_atx_heading(Scanner *s, TSLexer *lexer,
const bool *valid_symbols) {
if (valid_symbols[ATX_H1_MARKER] && s->indentation <= 3) {
mark_end(s, lexer);
uint16_t level = 0;
while (lexer->lookahead == '#' && level <= 6) {
advance(s, lexer);
level++;
}
if (level <= 6 &&
(lexer->lookahead == ' ' || lexer->lookahead == '\t' ||
lexer->lookahead == '\n' || lexer->lookahead == '\r')) {
lexer->result_symbol = ATX_H1_MARKER + (level - 1);
s->indentation = 0;
mark_end(s, lexer);
return true;
}
}
return false;
}
static bool parse_setext_underline(Scanner *s, TSLexer *lexer,
const bool *valid_symbols) {
if (valid_symbols[SETEXT_H1_UNDERLINE] &&
s->matched == s->open_blocks.size) {
mark_end(s, lexer);
while (lexer->lookahead == '=') {
advance(s, lexer);
}
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
advance(s, lexer);
}
if (lexer->lookahead == '\n' || lexer->lookahead == '\r') {
lexer->result_symbol = SETEXT_H1_UNDERLINE;
mark_end(s, lexer);
return true;
}
}
return false;
}
static bool parse_plus(Scanner *s, TSLexer *lexer, const bool *valid_symbols) {
if (s->indentation <= 3 &&
(valid_symbols[LIST_MARKER_PLUS] ||
valid_symbols[LIST_MARKER_PLUS_DONT_INTERRUPT] ||
valid_symbols[PLUS_METADATA])) {
advance(s, lexer);
if (valid_symbols[PLUS_METADATA] && lexer->lookahead == '+') {
advance(s, lexer);
if (lexer->lookahead != '+') {
return false;
}
advance(s, lexer);
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
advance(s, lexer);
}
if (lexer->lookahead != '\n' && lexer->lookahead != '\r') {
return false;
}
for (;;) {
// advance over newline
if (lexer->lookahead == '\r') {
advance(s, lexer);
if (lexer->lookahead == '\n') {
advance(s, lexer);
}
} else {
advance(s, lexer);
}
// check for pluses
size_t plus_count = 0;
while (lexer->lookahead == '+') {
plus_count++;
advance(s, lexer);
}
if (plus_count == 3) {
// if exactly 3 check if next symbol (after eventual
// whitespace) is newline
while (lexer->lookahead == ' ' ||
lexer->lookahead == '\t') {
advance(s, lexer);
}
if (lexer->lookahead == '\r' || lexer->lookahead == '\n') {
// if so also consume newline
if (lexer->lookahead == '\r') {
advance(s, lexer);
if (lexer->lookahead == '\n') {
advance(s, lexer);
}
} else {
advance(s, lexer);
}
mark_end(s, lexer);
lexer->result_symbol = PLUS_METADATA;
return true;
}
}
// otherwise consume rest of line
while (lexer->lookahead != '\n' && lexer->lookahead != '\r' &&
!lexer->eof(lexer)) {
advance(s, lexer);
}
// if end of file is reached, then this is not metadata
if (lexer->eof(lexer)) {
break;
}
}
} else {
uint8_t extra_indentation = 0;
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
extra_indentation += advance(s, lexer);
}
bool dont_interrupt = false;
if (lexer->lookahead == '\r' || lexer->lookahead == '\n') {
extra_indentation = 1;
dont_interrupt = true;
}
dont_interrupt =
dont_interrupt && s->matched == s->open_blocks.size;
if (extra_indentation >= 1 &&
(dont_interrupt ? valid_symbols[LIST_MARKER_PLUS_DONT_INTERRUPT]
: valid_symbols[LIST_MARKER_PLUS])) {
lexer->result_symbol = dont_interrupt
? LIST_MARKER_PLUS_DONT_INTERRUPT
: LIST_MARKER_PLUS;
extra_indentation--;
if (extra_indentation <= 3) {
extra_indentation += s->indentation;
s->indentation = 0;
} else {
uint8_t temp = s->indentation;
s->indentation = extra_indentation;
extra_indentation = temp;
}
if (!s->simulate)
push_block(s, (Block)(LIST_ITEM + extra_indentation));
return true;
}
}
}
return false;
}
static bool parse_ordered_list_marker(Scanner *s, TSLexer *lexer,
const bool *valid_symbols) {
if (s->indentation <= 3 &&
(valid_symbols[LIST_MARKER_PARENTHESIS] ||
valid_symbols[LIST_MARKER_DOT] ||
valid_symbols[LIST_MARKER_PARENTHESIS_DONT_INTERRUPT] ||
valid_symbols[LIST_MARKER_DOT_DONT_INTERRUPT])) {
size_t digits = 1;
bool dont_interrupt = lexer->lookahead != '1';
advance(s, lexer);
while (isdigit(lexer->lookahead)) {
dont_interrupt = true;
digits++;
advance(s, lexer);
}
if (digits >= 1 && digits <= 9) {
bool dot = false;
bool parenthesis = false;
if (lexer->lookahead == '.') {
advance(s, lexer);
dot = true;
} else if (lexer->lookahead == ')') {
advance(s, lexer);
parenthesis = true;
}
if (dot || parenthesis) {
uint8_t extra_indentation = 0;
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
extra_indentation += advance(s, lexer);
}
bool line_end =
lexer->lookahead == '\n' || lexer->lookahead == '\r';
if (line_end) {
extra_indentation = 1;
dont_interrupt = true;
}
dont_interrupt =
dont_interrupt && s->matched == s->open_blocks.size;
if (extra_indentation >= 1 &&
(dot ? (dont_interrupt
? valid_symbols[LIST_MARKER_DOT_DONT_INTERRUPT]
: valid_symbols[LIST_MARKER_DOT])
: (dont_interrupt
? valid_symbols
[LIST_MARKER_PARENTHESIS_DONT_INTERRUPT]
: valid_symbols[LIST_MARKER_PARENTHESIS]))) {
lexer->result_symbol =
dot ? LIST_MARKER_DOT : LIST_MARKER_PARENTHESIS;
extra_indentation--;
if (extra_indentation <= 3) {
extra_indentation += s->indentation;
s->indentation = 0;
} else {
uint8_t temp = s->indentation;
s->indentation = extra_indentation;
extra_indentation = temp;
}
if (!s->simulate)
push_block(
s, (Block)(LIST_ITEM + extra_indentation + digits));
return true;
}
}
}
}
return false;
}
static bool parse_minus(Scanner *s, TSLexer *lexer, const bool *valid_symbols) {
if (s->indentation <= 3 &&
(valid_symbols[LIST_MARKER_MINUS] ||
valid_symbols[LIST_MARKER_MINUS_DONT_INTERRUPT] ||
valid_symbols[SETEXT_H2_UNDERLINE] || valid_symbols[THEMATIC_BREAK] ||
valid_symbols[MINUS_METADATA])) {
mark_end(s, lexer);
bool whitespace_after_minus = false;
bool minus_after_whitespace = false;
size_t minus_count = 0;
uint8_t extra_indentation = 0;
for (;;) {
if (lexer->lookahead == '-') {
if (minus_count == 1 && extra_indentation >= 1) {
mark_end(s, lexer);
}
minus_count++;
advance(s, lexer);
minus_after_whitespace = whitespace_after_minus;
} else if (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
if (minus_count == 1) {
extra_indentation += advance(s, lexer);
} else {
advance(s, lexer);
}
whitespace_after_minus = true;
} else {
break;
}
}
bool line_end = lexer->lookahead == '\n' || lexer->lookahead == '\r';
bool dont_interrupt = false;
if (minus_count == 1 && line_end) {
extra_indentation = 1;
dont_interrupt = true;
}
dont_interrupt = dont_interrupt && s->matched == s->open_blocks.size;
bool thematic_break = minus_count >= 3 && line_end;
bool underline =
minus_count >= 1 && !minus_after_whitespace && line_end &&
s->matched ==
s->open_blocks
.size; // setext heading can not break lazy continuation
bool list_marker_minus = minus_count >= 1 && extra_indentation >= 1;
bool success = false;
if (valid_symbols[SETEXT_H2_UNDERLINE] && underline) {
lexer->result_symbol = SETEXT_H2_UNDERLINE;
mark_end(s, lexer);
s->indentation = 0;
success = true;
} else if (valid_symbols[THEMATIC_BREAK] &&
thematic_break) { // underline is false if list_marker_minus
// is true
lexer->result_symbol = THEMATIC_BREAK;
mark_end(s, lexer);
s->indentation = 0;
success = true;
} else if ((dont_interrupt
? valid_symbols[LIST_MARKER_MINUS_DONT_INTERRUPT]
: valid_symbols[LIST_MARKER_MINUS]) &&
list_marker_minus) {
if (minus_count == 1) {
mark_end(s, lexer);
}
extra_indentation--;
if (extra_indentation <= 3) {
extra_indentation += s->indentation;
s->indentation = 0;
} else {
uint8_t temp = s->indentation;
s->indentation = extra_indentation;
extra_indentation = temp;
}
if (!s->simulate)
push_block(s, (Block)(LIST_ITEM + extra_indentation));
lexer->result_symbol = dont_interrupt
? LIST_MARKER_MINUS_DONT_INTERRUPT
: LIST_MARKER_MINUS;
return true;
}
if (minus_count == 3 && (!minus_after_whitespace) && line_end &&
valid_symbols[MINUS_METADATA]) {
for (;;) {
// advance over newline
if (lexer->lookahead == '\r') {
advance(s, lexer);
if (lexer->lookahead == '\n') {
advance(s, lexer);
}
} else {
advance(s, lexer);
}
// check for minuses
minus_count = 0;
while (lexer->lookahead == '-') {
minus_count++;
advance(s, lexer);
}
if (minus_count == 3) {
// if exactly 3 check if next symbol (after eventual
// whitespace) is newline
while (lexer->lookahead == ' ' ||
lexer->lookahead == '\t') {
advance(s, lexer);
}
if (lexer->lookahead == '\r' || lexer->lookahead == '\n') {
// if so also consume newline
if (lexer->lookahead == '\r') {
advance(s, lexer);
if (lexer->lookahead == '\n') {
advance(s, lexer);
}
} else {
advance(s, lexer);
}
mark_end(s, lexer);
lexer->result_symbol = MINUS_METADATA;
return true;
}
}
// otherwise consume rest of line
while (lexer->lookahead != '\n' && lexer->lookahead != '\r' &&
!lexer->eof(lexer)) {
advance(s, lexer);
}
// if end of file is reached, then this is not metadata
if (lexer->eof(lexer)) {
break;
}
}
}
if (success) {
return true;
}
}
return false;
}
static bool parse_html_block(Scanner *s, TSLexer *lexer,
const bool *valid_symbols) {
if (!(valid_symbols[HTML_BLOCK_1_START] ||
valid_symbols[HTML_BLOCK_1_END] ||
valid_symbols[HTML_BLOCK_2_START] ||
valid_symbols[HTML_BLOCK_3_START] ||
valid_symbols[HTML_BLOCK_4_START] ||
valid_symbols[HTML_BLOCK_5_START] ||
valid_symbols[HTML_BLOCK_6_START] ||
valid_symbols[HTML_BLOCK_7_START])) {
return false;
}
advance(s, lexer);
if (lexer->lookahead == '?' && valid_symbols[HTML_BLOCK_3_START]) {
advance(s, lexer);
lexer->result_symbol = HTML_BLOCK_3_START;
if (!s->simulate)
push_block(s, ANONYMOUS);
return true;
}
if (lexer->lookahead == '!') {
// could be block 2
advance(s, lexer);
if (lexer->lookahead == '-') {
advance(s, lexer);
if (lexer->lookahead == '-' && valid_symbols[HTML_BLOCK_2_START]) {
advance(s, lexer);
lexer->result_symbol = HTML_BLOCK_2_START;
if (!s->simulate)
push_block(s, ANONYMOUS);
return true;
}
} else if ('A' <= lexer->lookahead && lexer->lookahead <= 'Z' &&
valid_symbols[HTML_BLOCK_4_START]) {
advance(s, lexer);
lexer->result_symbol = HTML_BLOCK_4_START;
if (!s->simulate)
push_block(s, ANONYMOUS);
return true;
} else if (lexer->lookahead == '[') {
advance(s, lexer);
if (lexer->lookahead == 'C') {
advance(s, lexer);
if (lexer->lookahead == 'D') {
advance(s, lexer);
if (lexer->lookahead == 'A') {
advance(s, lexer);
if (lexer->lookahead == 'T') {
advance(s, lexer);
if (lexer->lookahead == 'A') {
advance(s, lexer);
if (lexer->lookahead == '[' &&
valid_symbols[HTML_BLOCK_5_START]) {
advance(s, lexer);
lexer->result_symbol = HTML_BLOCK_5_START;
if (!s->simulate)
push_block(s, ANONYMOUS);
return true;
}
}
}
}
}
}
}
}
bool starting_slash = lexer->lookahead == '/';
if (starting_slash) {
advance(s, lexer);
}
char name[11];
size_t name_length = 0;
while (iswalpha((wint_t)lexer->lookahead)) {
if (name_length < 10) {
name[name_length++] = (char)towlower((wint_t)lexer->lookahead);
} else {
name_length = 12;
}
advance(s, lexer);
}
if (name_length == 0) {
return false;
}
bool tag_closed = false;
if (name_length < 11) {
name[name_length] = 0;
bool next_symbol_valid =
lexer->lookahead == ' ' || lexer->lookahead == '\t' ||
lexer->lookahead == '\n' || lexer->lookahead == '\r' ||
lexer->lookahead == '>';
if (next_symbol_valid) {
// try block 1 names
for (size_t i = 0; i < NUM_HTML_TAG_NAMES_RULE_1; i++) {
if (strcmp(name, HTML_TAG_NAMES_RULE_1[i]) == 0) {
if (starting_slash) {
if (valid_symbols[HTML_BLOCK_1_END]) {
lexer->result_symbol = HTML_BLOCK_1_END;
return true;
}
} else if (valid_symbols[HTML_BLOCK_1_START]) {
lexer->result_symbol = HTML_BLOCK_1_START;
if (!s->simulate)
push_block(s, ANONYMOUS);
return true;
}
}
}
}
if (!next_symbol_valid && lexer->lookahead == '/') {
advance(s, lexer);
if (lexer->lookahead == '>') {
advance(s, lexer);
tag_closed = true;
}
}
if (next_symbol_valid || tag_closed) {
// try block 2 names
for (size_t i = 0; i < NUM_HTML_TAG_NAMES_RULE_7; i++) {
if (strcmp(name, HTML_TAG_NAMES_RULE_7[i]) == 0 &&
valid_symbols[HTML_BLOCK_6_START]) {
lexer->result_symbol = HTML_BLOCK_6_START;
if (!s->simulate)
push_block(s, ANONYMOUS);
return true;
}
}
}
}
if (!valid_symbols[HTML_BLOCK_7_START]) {
return false;
}
if (!tag_closed) {
// tag name (continued)
while (iswalnum((wint_t)lexer->lookahead) || lexer->lookahead == '-') {
advance(s, lexer);
}
if (!starting_slash) {
// attributes
bool had_whitespace = false;
for (;;) {
// whitespace
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
had_whitespace = true;
advance(s, lexer);
}
if (lexer->lookahead == '/') {
advance(s, lexer);
break;
}
if (lexer->lookahead == '>') {
break;
}
// attribute name
if (!had_whitespace) {
return false;
}
if (!iswalpha((wint_t)lexer->lookahead) &&
lexer->lookahead != '_' && lexer->lookahead != ':') {
return false;
}
had_whitespace = false;
advance(s, lexer);
while (iswalnum((wint_t)lexer->lookahead) ||
lexer->lookahead == '_' || lexer->lookahead == '.' ||
lexer->lookahead == ':' || lexer->lookahead == '-') {
advance(s, lexer);
}
// attribute value specification
// optional whitespace
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
had_whitespace = true;
advance(s, lexer);
}
// =
if (lexer->lookahead == '=') {
advance(s, lexer);
had_whitespace = false;
// optional whitespace
while (lexer->lookahead == ' ' ||
lexer->lookahead == '\t') {
advance(s, lexer);
}
// attribute value
if (lexer->lookahead == '\'' || lexer->lookahead == '"') {
char delimiter = (char)lexer->lookahead;
advance(s, lexer);
while (lexer->lookahead != delimiter &&
lexer->lookahead != '\n' &&
lexer->lookahead != '\r' && !lexer->eof(lexer)) {
advance(s, lexer);
}
if (lexer->lookahead != delimiter) {
return false;
}
advance(s, lexer);
} else {
// unquoted attribute value
bool had_one = false;
while (lexer->lookahead != ' ' &&
lexer->lookahead != '\t' &&
lexer->lookahead != '"' &&
lexer->lookahead != '\'' &&
lexer->lookahead != '=' &&
lexer->lookahead != '<' &&
lexer->lookahead != '>' &&
lexer->lookahead != '`' &&
lexer->lookahead != '\n' &&
lexer->lookahead != '\r' && !lexer->eof(lexer)) {
advance(s, lexer);
had_one = true;
}
if (!had_one) {
return false;
}
}
}
}
} else {
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
advance(s, lexer);
}
}
if (lexer->lookahead != '>') {
return false;
}
advance(s, lexer);
}
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
advance(s, lexer);
}
if (lexer->lookahead == '\r' || lexer->lookahead == '\n') {
lexer->result_symbol = HTML_BLOCK_7_START;
if (!s->simulate)
push_block(s, ANONYMOUS);
return true;
}
return false;
}
static bool parse_pipe_table(Scanner *s, TSLexer *lexer,
const bool *valid_symbols) {
// unused
(void)(valid_symbols);
// PIPE_TABLE_START is zero width
mark_end(s, lexer);
// count number of cells
size_t cell_count = 0;
// also remember if we see starting and ending pipes, as empty headers have
// to have both
bool starting_pipe = false;
bool ending_pipe = false;
bool empty = true;
if (lexer->lookahead == '|') {
starting_pipe = true;
advance(s, lexer);
}
while (lexer->lookahead != '\r' && lexer->lookahead != '\n' &&
!lexer->eof(lexer)) {
if (lexer->lookahead == '|') {
cell_count++;
ending_pipe = true;
advance(s, lexer);
} else {
if (lexer->lookahead != ' ' && lexer->lookahead != '\t') {
ending_pipe = false;
}
if (lexer->lookahead == '\\') {
advance(s, lexer);
if (is_punctuation((char)lexer->lookahead)) {
advance(s, lexer);
}
} else {
advance(s, lexer);
}
}
}
if (empty && cell_count == 0 && !(starting_pipe && ending_pipe)) {
return false;
}
if (!ending_pipe) {
cell_count++;
}
// check the following line for a delimiter row
// parse a newline
if (lexer->lookahead == '\n') {
advance(s, lexer);
} else if (lexer->lookahead == '\r') {
advance(s, lexer);
if (lexer->lookahead == '\n') {
advance(s, lexer);
}
} else {
return false;
}
s->indentation = 0;
s->column = 0;
for (;;) {
if (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
s->indentation += advance(s, lexer);
} else {
break;
}
}
s->simulate = true;
uint8_t matched_temp = 0;
while (matched_temp < (uint8_t)s->open_blocks.size) {
if (match(s, lexer, s->open_blocks.items[matched_temp])) {
matched_temp++;
} else {
return false;
}
}
// check if delimiter row has the same number of cells and at least one pipe
size_t delimiter_cell_count = 0;
if (lexer->lookahead == '|') {
advance(s, lexer);
}
for (;;) {
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
advance(s, lexer);
}
if (lexer->lookahead == '|') {
delimiter_cell_count++;
advance(s, lexer);
continue;
}
if (lexer->lookahead == ':') {
advance(s, lexer);
if (lexer->lookahead != '-') {
return false;
}
}
bool had_one_minus = false;
while (lexer->lookahead == '-') {
had_one_minus = true;
advance(s, lexer);
}
if (had_one_minus) {
delimiter_cell_count++;
}
if (lexer->lookahead == ':') {
if (!had_one_minus) {
return false;
}
advance(s, lexer);
}
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
advance(s, lexer);
}
if (lexer->lookahead == '|') {
if (!had_one_minus) {
delimiter_cell_count++;
}
advance(s, lexer);
continue;
}
if (lexer->lookahead != '\r' && lexer->lookahead != '\n') {
return false;
} else {
break;
}
}
// if the cell counts are not equal then this is not a table
if (cell_count != delimiter_cell_count) {
return false;
}
lexer->result_symbol = PIPE_TABLE_START;
return true;
}
static bool scan(Scanner *s, TSLexer *lexer, const bool *valid_symbols) {
// A normal tree-sitter rule decided that the current branch is invalid and
// now "requests" an error to stop the branch
if (valid_symbols[TRIGGER_ERROR]) {
return error(lexer);
}
// Close the inner most block after the next line break as requested. See
// `$._close_block` in grammar.js
if (valid_symbols[CLOSE_BLOCK]) {
s->state |= STATE_CLOSE_BLOCK;
lexer->result_symbol = CLOSE_BLOCK;
return true;
}
// if we are at the end of the file and there are still open blocks close
// them all
if (lexer->eof(lexer)) {
if (valid_symbols[TOKEN_EOF]) {
lexer->result_symbol = TOKEN_EOF;
return true;
}
if (s->open_blocks.size > 0) {
lexer->result_symbol = BLOCK_CLOSE;
if (!s->simulate)
pop_block(s);
return true;
}
return false;
}
if (!(s->state & STATE_MATCHING)) {
// Parse any preceeding whitespace and remember its length. This makes a
// lot of parsing quite a bit easier.
for (;;) {
if (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
s->indentation += advance(s, lexer);
} else {
break;
}
}
// We are not matching. This is where the parsing logic for most
// "normal" token is. Most importantly parsing logic for the start of
// new blocks.
if (valid_symbols[INDENTED_CHUNK_START] &&
!valid_symbols[NO_INDENTED_CHUNK]) {
if (s->indentation >= 4 && lexer->lookahead != '\n' &&
lexer->lookahead != '\r') {
lexer->result_symbol = INDENTED_CHUNK_START;
if (!s->simulate)
push_block(s, INDENTED_CODE_BLOCK);
s->indentation -= 4;
return true;
}
}
// Decide which tokens to consider based on the first non-whitespace
// character
switch (lexer->lookahead) {
case '\r':
case '\n':
if (valid_symbols[BLANK_LINE_START]) {
// A blank line token is actually just 0 width, so do not
// consume the characters
lexer->result_symbol = BLANK_LINE_START;
return true;
}
break;
case '`':
// A backtick could mark the beginning or ending of a fenced
// code block.
return parse_fenced_code_block(s, '`', lexer, valid_symbols);
case '~':
// A tilde could mark the beginning or ending of a fenced code
// block.
return parse_fenced_code_block(s, '~', lexer, valid_symbols);
case '*':
// A star could either mark a list item or a thematic break.
// This code is similar to the code for '_' and '+'.
return parse_star(s, lexer, valid_symbols);
case '_':
return parse_thematic_break_underscore(s, lexer, valid_symbols);
case '>':
// A '>' could mark the beginning of a block quote
return parse_block_quote(s, lexer, valid_symbols);
case '#':
// A '#' could mark a atx heading
return parse_atx_heading(s, lexer, valid_symbols);
case '=':
// A '=' could mark a setext underline
return parse_setext_underline(s, lexer, valid_symbols);
case '+':
// A '+' could be a list marker
return parse_plus(s, lexer, valid_symbols);
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
// A number could be a list marker (if followed by a dot or a
// parenthesis)
return parse_ordered_list_marker(s, lexer, valid_symbols);
case '-':
// A minus could mark a list marker, a thematic break or a
// setext underline
return parse_minus(s, lexer, valid_symbols);
case '<':
// A < could mark the beginning of a html block
return parse_html_block(s, lexer, valid_symbols);
}
if (lexer->lookahead != '\r' && lexer->lookahead != '\n' &&
valid_symbols[PIPE_TABLE_START]) {
return parse_pipe_table(s, lexer, valid_symbols);
}
} else { // we are in the state of trying to match all currently open blocks
bool partial_success = false;
while (s->matched < (uint8_t)s->open_blocks.size) {
if (s->matched == (uint8_t)s->open_blocks.size - 1 &&
(s->state & STATE_CLOSE_BLOCK)) {
if (!partial_success)
s->state &= ~STATE_CLOSE_BLOCK;
break;
}
if (match(s, lexer, s->open_blocks.items[s->matched])) {
partial_success = true;
s->matched++;
} else {
if (s->state & STATE_WAS_SOFT_LINE_BREAK) {
s->state &= (~STATE_MATCHING);
}
break;
}
}
if (partial_success) {
if (s->matched == s->open_blocks.size) {
s->state &= (~STATE_MATCHING);
}
lexer->result_symbol = BLOCK_CONTINUATION;
return true;
}
if (!(s->state & STATE_WAS_SOFT_LINE_BREAK)) {
lexer->result_symbol = BLOCK_CLOSE;
pop_block(s);
if (s->matched == s->open_blocks.size) {
s->state &= (~STATE_MATCHING);
}
return true;
}
}
// The parser just encountered a line break. Setup the state correspondingly
if ((valid_symbols[LINE_ENDING] || valid_symbols[SOFT_LINE_ENDING] ||
valid_symbols[PIPE_TABLE_LINE_ENDING]) &&
(lexer->lookahead == '\n' || lexer->lookahead == '\r')) {
if (lexer->lookahead == '\r') {
advance(s, lexer);
if (lexer->lookahead == '\n') {
advance(s, lexer);
}
} else {
advance(s, lexer);
}
s->indentation = 0;
s->column = 0;
if (!(s->state & STATE_CLOSE_BLOCK) &&
(valid_symbols[SOFT_LINE_ENDING] ||
valid_symbols[PIPE_TABLE_LINE_ENDING])) {
lexer->mark_end(lexer);
for (;;) {
if (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
s->indentation += advance(s, lexer);
} else {
break;
}
}
s->simulate = true;
uint8_t matched_temp = s->matched;
s->matched = 0;
bool one_will_be_matched = false;
while (s->matched < (uint8_t)s->open_blocks.size) {
if (match(s, lexer, s->open_blocks.items[s->matched])) {
s->matched++;
one_will_be_matched = true;
} else {
break;
}
}
bool all_will_be_matched = s->matched == s->open_blocks.size;
if (!lexer->eof(lexer) &&
!scan(s, lexer, paragraph_interrupt_symbols)) {
s->matched = matched_temp;
// If the last line break ended a paragraph and no new block
// opened, the last line break should have been a soft line
// break Reset the counter for matched blocks
s->matched = 0;
s->indentation = 0;
s->column = 0;
// If there is at least one open block, we should be in the
// matching state. Also set the matching flag if a
// `$._soft_line_break_marker` can be emitted so it does get
// emitted.
if (one_will_be_matched) {
s->state |= STATE_MATCHING;
} else {
s->state &= (~STATE_MATCHING);
}
if (valid_symbols[PIPE_TABLE_LINE_ENDING]) {
if (all_will_be_matched) {
lexer->result_symbol = PIPE_TABLE_LINE_ENDING;
return true;
}
} else {
lexer->result_symbol = SOFT_LINE_ENDING;
// reset some state variables
s->state |= STATE_WAS_SOFT_LINE_BREAK;
return true;
}
} else {
s->matched = matched_temp;
}
s->indentation = 0;
s->column = 0;
}
if (valid_symbols[LINE_ENDING]) {
// If the last line break ended a paragraph and no new block opened,
// the last line break should have been a soft line break Reset the
// counter for matched blocks
s->matched = 0;
// If there is at least one open block, we should be in the matching
// state. Also set the matching flag if a
// `$._soft_line_break_marker` can be emitted so it does get
// emitted.
if (s->open_blocks.size > 0) {
s->state |= STATE_MATCHING;
} else {
s->state &= (~STATE_MATCHING);
}
// reset some state variables
s->state &= (~STATE_WAS_SOFT_LINE_BREAK);
lexer->result_symbol = LINE_ENDING;
return true;
}
}
return false;
}
void *tree_sitter_markdown_external_scanner_create(void) {
Scanner *s = (Scanner *)malloc(sizeof(Scanner));
s->open_blocks.items = (Block *)calloc(1, sizeof(Block));
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
_Static_assert(ATX_H6_MARKER == ATX_H1_MARKER + 5, "");
#else
assert(ATX_H6_MARKER == ATX_H1_MARKER + 5);
#endif
deserialize(s, NULL, 0);
return s;
}
bool tree_sitter_markdown_external_scanner_scan(void *payload, TSLexer *lexer,
const bool *valid_symbols) {
Scanner *scanner = (Scanner *)payload;
scanner->simulate = false;
return scan(scanner, lexer, valid_symbols);
}
unsigned tree_sitter_markdown_external_scanner_serialize(void *payload,
char *buffer) {
Scanner *scanner = (Scanner *)payload;
return serialize(scanner, buffer);
}
void tree_sitter_markdown_external_scanner_deserialize(void *payload,
const char *buffer,
unsigned length) {
Scanner *scanner = (Scanner *)payload;
deserialize(scanner, buffer, length);
}
void tree_sitter_markdown_external_scanner_destroy(void *payload) {
Scanner *scanner = (Scanner *)payload;
free(scanner->open_blocks.items);
free(scanner);
}
```
--------------------------------------------------------------------------------
/src/core/editor.cpp:
--------------------------------------------------------------------------------
```cpp
#include "editor.h"
// #include "src/ui/colors.h"
#include "src/core/config_manager.h"
#include "src/ui/style_manager.h"
#include <algorithm>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#ifdef _WIN32
#include <curses.h>
#else
#include <ncurses.h>
#endif
#include <iostream>
#include <sstream>
#include <string>
#include <utility>
// Windows-specific mouse codes
#ifndef BUTTON4_PRESSED
#define BUTTON4_PRESSED 0x00200000L
#endif
#ifndef BUTTON5_PRESSED
#define BUTTON5_PRESSED 0x00100000L
#endif
// =================================================================
// Constructor
// =================================================================
Editor::Editor(SyntaxHighlighter *highlighter) : syntaxHighlighter(highlighter)
{
tabSize = ConfigManager::getTabSize();
}
EditorSnapshot Editor::captureSnapshot() const
{
EditorSnapshot snap;
snap.lineCount = buffer.getLineCount();
snap.cursorLine = cursorLine;
snap.cursorCol = cursorCol;
snap.viewportTop = viewportTop;
snap.viewportLeft = viewportLeft;
snap.bufferSize = buffer.size();
if (snap.lineCount > 0)
{
snap.firstLine = buffer.getLine(0);
snap.lastLine = buffer.getLine(snap.lineCount - 1);
if (cursorLine < snap.lineCount)
{
snap.cursorLineContent = buffer.getLine(cursorLine);
}
}
return snap;
}
ValidationResult Editor::validateState(const std::string &context) const
{
// Check buffer is not empty
if (buffer.getLineCount() == 0)
{
return ValidationResult("Buffer has 0 lines at: " + context);
}
// Check cursor line bounds
if (cursorLine < 0 || cursorLine >= buffer.getLineCount())
{
std::ostringstream oss;
oss << "Cursor line " << cursorLine << " out of bounds [0, "
<< buffer.getLineCount() - 1 << "] at: " << context;
return ValidationResult(oss.str());
}
// Check cursor column bounds
std::string line = buffer.getLine(cursorLine);
if (cursorCol < 0 || cursorCol > static_cast<int>(line.length()))
{
std::ostringstream oss;
oss << "Cursor col " << cursorCol << " out of bounds [0, " << line.length()
<< "] at: " << context;
return ValidationResult(oss.str());
}
// Check viewport bounds
if (viewportTop < 0)
{
return ValidationResult("Viewport top negative at: " + context);
}
if (viewportLeft < 0)
{
return ValidationResult("Viewport left negative at: " + context);
}
// Check viewport can contain cursor
if (cursorLine < viewportTop)
{
std::ostringstream oss;
oss << "Cursor line " << cursorLine << " above viewport " << viewportTop
<< " at: " + context;
return ValidationResult(oss.str());
}
return ValidationResult(); // All valid
}
// Compare two snapshots and report differences
std::string Editor::compareSnapshots(const EditorSnapshot &before,
const EditorSnapshot &after) const
{
std::ostringstream oss;
if (before.lineCount != after.lineCount)
{
oss << "LineCount: " << before.lineCount << " -> " << after.lineCount
<< "\n";
}
if (before.cursorLine != after.cursorLine)
{
oss << "CursorLine: " << before.cursorLine << " -> " << after.cursorLine
<< "\n";
}
if (before.cursorCol != after.cursorCol)
{
oss << "CursorCol: " << before.cursorCol << " -> " << after.cursorCol
<< "\n";
}
if (before.bufferSize != after.bufferSize)
{
oss << "BufferSize: " << before.bufferSize << " -> " << after.bufferSize
<< "\n";
}
if (before.cursorLineContent != after.cursorLineContent)
{
oss << "CursorLine content changed\n";
oss << " Before: '" << before.cursorLineContent << "'\n";
oss << " After: '" << after.cursorLineContent << "'\n";
}
return oss.str();
}
void Editor::reloadConfig()
{
tabSize = ConfigManager::getTabSize();
// Trigger redisplay to reflect changes
}
// =================================================================
// Mode Management
// =================================================================
// =================================================================
// Private Helper Methods (from original code)
// =================================================================
std::string Editor::expandTabs(const std::string &line, int tabSize)
{
std::string result;
for (char c : line)
{
if (c == '\t')
{
int spacesToAdd = tabSize - (result.length() % tabSize);
result.append(spacesToAdd, ' ');
}
else if (c >= 32 && c <= 126)
{
result += c;
}
else
{
result += ' ';
}
}
return result;
}
std::string Editor::getFileExtension()
{
if (filename.empty())
return "";
size_t dot = filename.find_last_of(".");
if (dot == std::string::npos)
return "";
std::string ext = filename.substr(dot + 1);
std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
return ext;
}
bool Editor::isPositionSelected(int line, int col)
{
if (!hasSelection && !isSelecting)
return false;
int startL = selectionStartLine;
int startC = selectionStartCol;
int endL = selectionEndLine;
int endC = selectionEndCol;
if (startL > endL || (startL == endL && startC > endC))
{
std::swap(startL, endL);
std::swap(startC, endC);
}
if (line < startL || line > endL)
return false;
if (startL == endL)
{
return col >= startC && col < endC;
}
else if (line == startL)
{
return col >= startC;
}
else if (line == endL)
{
return col < endC;
}
else
{
return true;
}
}
void Editor::positionCursor()
{
int rows, cols;
getmaxyx(stdscr, rows, cols);
int screenRow = cursorLine - viewportTop;
if (screenRow >= 0 && screenRow < viewportHeight)
{
bool show_line_numbers = ConfigManager::getLineNumbers();
int lineNumWidth =
show_line_numbers ? std::to_string(buffer.getLineCount()).length() : 0;
int contentStartCol = show_line_numbers ? (lineNumWidth + 3) : 0;
int screenCol = contentStartCol + cursorCol - viewportLeft;
if (screenCol >= contentStartCol && screenCol < cols)
{
move(screenRow, screenCol);
}
else
{
move(screenRow, contentStartCol);
}
}
// REMOVED: All #ifdef _WIN32 refresh() calls
}
bool Editor::mouseToFilePos(int mouseRow, int mouseCol, int &fileRow,
int &fileCol)
{
int rows, cols;
getmaxyx(stdscr, rows, cols);
if (mouseRow >= rows - 1)
return false;
bool show_line_numbers = ConfigManager::getLineNumbers();
int lineNumWidth =
show_line_numbers ? std::to_string(buffer.getLineCount()).length() : 0;
int contentStartCol = show_line_numbers ? (lineNumWidth + 3) : 0;
if (mouseCol < contentStartCol)
{
mouseCol = contentStartCol;
}
fileRow = viewportTop + mouseRow;
if (fileRow < 0)
fileRow = 0;
if (fileRow >= buffer.getLineCount())
fileRow = buffer.getLineCount() - 1;
fileCol = viewportLeft + (mouseCol - contentStartCol);
if (fileCol < 0)
fileCol = 0;
return true;
}
void Editor::updateCursorAndViewport(int newLine, int newCol)
{
cursorLine = newLine;
int currentTabSize = ConfigManager::getTabSize();
std::string expandedLine =
expandTabs(buffer.getLine(cursorLine), currentTabSize);
cursorCol = std::min(newCol, static_cast<int>(expandedLine.length()));
if (cursorLine < viewportTop)
{
viewportTop = cursorLine;
}
else if (cursorLine >= viewportTop + viewportHeight)
{
viewportTop = cursorLine - viewportHeight + 1;
}
int rows, cols;
getmaxyx(stdscr, rows, cols);
bool show_line_numbers = ConfigManager::getLineNumbers();
int lineNumWidth =
show_line_numbers ? std::to_string(buffer.getLineCount()).length() : 0;
int contentWidth = cols - (show_line_numbers ? (lineNumWidth + 3) : 0);
if (cursorCol < viewportLeft)
{
viewportLeft = cursorCol;
}
else if (cursorCol >= viewportLeft + contentWidth)
{
viewportLeft = cursorCol - contentWidth + 1;
}
}
// =================================================================
// Public API Methods
// =================================================================
void Editor::setSyntaxHighlighter(SyntaxHighlighter *highlighter)
{
syntaxHighlighter = highlighter;
}
void Editor::display()
{
// Validate state
if (!validateEditorState())
{
validateCursorAndViewport();
if (!validateEditorState())
return;
}
int rows, cols;
getmaxyx(stdscr, rows, cols);
viewportHeight = rows - 1;
bool show_line_numbers = ConfigManager::getLineNumbers();
int lineNumWidth =
show_line_numbers ? std::to_string(buffer.getLineCount()).length() : 0;
int contentStartCol = show_line_numbers ? (lineNumWidth + 3) : 0;
int contentWidth = cols - contentStartCol;
int endLine = std::min(viewportTop + viewportHeight, buffer.getLineCount());
// OPTIMIZATION: Pre-mark viewport lines for priority parsing
if (syntaxHighlighter)
{
syntaxHighlighter->markViewportLines(viewportTop, endLine - 1);
}
// Pre-compute selection (unchanged)
bool hasActiveSelection = (hasSelection || isSelecting);
int sel_start_line = -1, sel_start_col = -1;
int sel_end_line = -1, sel_end_col = -1;
if (hasActiveSelection)
{
auto [start, end] = getNormalizedSelection();
sel_start_line = start.first;
sel_start_col = start.second;
sel_end_line = end.first;
sel_end_col = end.second;
}
int currentTabSize = ConfigManager::getTabSize();
// OPTIMIZATION: Batch render - minimize attribute changes
for (int i = viewportTop; i < endLine; i++)
{
int screenRow = i - viewportTop;
bool isCurrentLine = (cursorLine == i);
move(screenRow, 0);
attrset(COLOR_PAIR(0));
// Render line numbers
if (show_line_numbers)
{
int ln_colorPair = isCurrentLine ? 3 : 2;
attron(COLOR_PAIR(ln_colorPair));
printw("%*d ", lineNumWidth, i + 1);
attroff(COLOR_PAIR(ln_colorPair));
attron(COLOR_PAIR(4));
addch(' ');
attroff(COLOR_PAIR(4));
addch(' ');
}
// Get line content
std::string expandedLine = expandTabs(buffer.getLine(i), currentTabSize);
// OPTIMIZATION: Get highlighting spans (cached if available)
std::vector<ColorSpan> currentLineSpans;
if (syntaxHighlighter)
{
try
{
currentLineSpans =
syntaxHighlighter->getHighlightSpans(expandedLine, i, buffer);
}
catch (...)
{
currentLineSpans.clear();
}
}
// Render line content (unchanged logic, but faster due to cached spans)
bool lineHasSelection =
hasActiveSelection && i >= sel_start_line && i <= sel_end_line;
int current_span_idx = 0;
int num_spans = currentLineSpans.size();
for (int screenCol = 0; screenCol < contentWidth; screenCol++)
{
int fileCol = viewportLeft + screenCol;
bool charExists =
(fileCol >= 0 && fileCol < static_cast<int>(expandedLine.length()));
char ch = charExists ? expandedLine[fileCol] : ' ';
if (charExists && (ch < 32 || ch > 126))
ch = ' ';
// Selection check
bool isSelected = false;
if (lineHasSelection && charExists)
{
if (sel_start_line == sel_end_line)
{
isSelected = (fileCol >= sel_start_col && fileCol < sel_end_col);
}
else if (i == sel_start_line)
{
isSelected = (fileCol >= sel_start_col);
}
else if (i == sel_end_line)
{
isSelected = (fileCol < sel_end_col);
}
else
{
isSelected = true;
}
}
if (isSelected)
{
attron(COLOR_PAIR(14) | A_REVERSE);
addch(ch);
attroff(COLOR_PAIR(14) | A_REVERSE);
}
else
{
bool colorApplied = false;
if (charExists && num_spans > 0)
{
while (current_span_idx < num_spans &&
currentLineSpans[current_span_idx].end <= fileCol)
{
current_span_idx++;
}
if (current_span_idx < num_spans)
{
const auto &span = currentLineSpans[current_span_idx];
if (fileCol >= span.start && fileCol < span.end)
{
if (span.colorPair >= 0 && span.colorPair < COLOR_PAIRS)
{
attron(COLOR_PAIR(span.colorPair));
if (span.attribute != 0)
attron(span.attribute);
addch(ch);
if (span.attribute != 0)
attroff(span.attribute);
attroff(COLOR_PAIR(span.colorPair));
colorApplied = true;
}
}
}
}
if (!colorApplied)
{
attrset(COLOR_PAIR(0));
addch(ch);
}
}
}
attrset(COLOR_PAIR(0));
clrtoeol();
}
// Clear remaining lines
attrset(COLOR_PAIR(0));
for (int i = endLine - viewportTop; i < viewportHeight; i++)
{
move(i, 0);
clrtoeol();
}
drawStatusBar();
positionCursor();
}
void Editor::drawStatusBar()
{
int rows, cols;
getmaxyx(stdscr, rows, cols);
int statusRow = rows - 1;
move(statusRow, 0);
attrset(COLOR_PAIR(STATUS_BAR));
clrtoeol();
move(statusRow, 0);
attron(COLOR_PAIR(STATUS_BAR));
// Show filename
attron(COLOR_PAIR(STATUS_BAR_CYAN) | A_BOLD);
if (filename.empty())
{
printw("[No Name]");
}
else
{
size_t lastSlash = filename.find_last_of("/\\");
std::string displayName = (lastSlash != std::string::npos)
? filename.substr(lastSlash + 1)
: filename;
printw("%s", displayName.c_str());
}
attroff(COLOR_PAIR(STATUS_BAR_CYAN) | A_BOLD);
// Show modified indicator
if (isModified)
{
attron(COLOR_PAIR(STATUS_BAR_ACTIVE) | A_BOLD);
printw(" [+]");
attroff(COLOR_PAIR(STATUS_BAR_ACTIVE) | A_BOLD);
}
// Show file extension
std::string ext = getFileExtension();
if (!ext.empty())
{
attron(COLOR_PAIR(STATUS_BAR_ACTIVE));
printw(" [%s]", ext.c_str());
attroff(COLOR_PAIR(STATUS_BAR_ACTIVE));
}
// Right section with position info
char rightSection[256];
if (hasSelection)
{
auto [start, end] = getNormalizedSelection();
int startL = start.first, startC = start.second;
int endL = end.first, endC = end.second;
if (startL == endL)
{
int selectionSize = endC - startC;
snprintf(rightSection, sizeof(rightSection),
"[%d chars] %d:%d %d/%d %d%% ", selectionSize, cursorLine + 1,
cursorCol + 1, cursorLine + 1, buffer.getLineCount(),
buffer.getLineCount() == 0
? 0
: ((cursorLine + 1) * 100 / buffer.getLineCount()));
}
else
{
int lineCount = endL - startL + 1;
snprintf(rightSection, sizeof(rightSection),
"[%d lines] %d:%d %d/%d %d%% ", lineCount, cursorLine + 1,
cursorCol + 1, cursorLine + 1, buffer.getLineCount(),
buffer.getLineCount() == 0
? 0
: ((cursorLine + 1) * 100 / buffer.getLineCount()));
}
}
else
{
snprintf(rightSection, sizeof(rightSection), "%d:%d %d/%d %d%% ",
cursorLine + 1, cursorCol + 1, cursorLine + 1,
buffer.getLineCount(),
buffer.getLineCount() == 0
? 0
: ((cursorLine + 1) * 100 / buffer.getLineCount()));
}
int rightLen = strlen(rightSection);
int currentPos = getcurx(stdscr);
int rightStart = cols - rightLen;
if (rightStart <= currentPos)
{
rightStart = currentPos + 2;
}
// Fill middle space
attron(COLOR_PAIR(STATUS_BAR));
for (int i = currentPos; i < rightStart && i < cols; i++)
{
move(statusRow, i);
addch(' ');
}
// Right section
if (rightStart < cols)
{
move(statusRow, rightStart);
attron(COLOR_PAIR(STATUS_BAR_YELLOW) | A_BOLD);
printw("%s", rightSection);
attroff(COLOR_PAIR(STATUS_BAR_YELLOW) | A_BOLD);
}
attroff(COLOR_PAIR(STATUS_BAR));
}
void Editor::handleResize()
{
int rows, cols;
getmaxyx(stdscr, rows, cols);
viewportHeight = rows - 1;
if (cursorLine >= viewportTop + viewportHeight)
{
viewportTop = cursorLine - viewportHeight + 1;
}
if (viewportTop < 0)
{
viewportTop = 0;
}
clear();
display();
wnoutrefresh(stdscr); // Mark stdscr as ready
doupdate(); // Execute the single, clean flush
}
void Editor::handleMouse(MEVENT &event)
{
if (event.bstate & BUTTON1_PRESSED)
{
int fileRow, fileCol;
if (mouseToFilePos(event.y, event.x, fileRow, fileCol))
{
// Start a new selection on mouse press
clearSelection();
isSelecting = true;
selectionStartLine = fileRow;
selectionStartCol = fileCol;
selectionEndLine = fileRow;
selectionEndCol = fileCol;
updateCursorAndViewport(fileRow, fileCol);
}
}
else if (event.bstate & BUTTON1_RELEASED)
{
if (isSelecting)
{
int fileRow, fileCol;
if (mouseToFilePos(event.y, event.x, fileRow, fileCol))
{
selectionEndLine = fileRow;
selectionEndCol = fileCol;
// Only keep selection if it's not just a click (start != end)
if (selectionStartLine != selectionEndLine ||
selectionStartCol != selectionEndCol)
{
hasSelection = true;
}
else
{
// Just a click, no drag - clear selection
clearSelection();
}
updateCursorAndViewport(fileRow, fileCol);
}
isSelecting = false;
}
}
else if ((event.bstate & REPORT_MOUSE_POSITION) && isSelecting)
{
// Mouse drag - extend selection
int fileRow, fileCol;
if (mouseToFilePos(event.y, event.x, fileRow, fileCol))
{
selectionEndLine = fileRow;
selectionEndCol = fileCol;
updateCursorAndViewport(fileRow, fileCol);
}
}
else if (event.bstate & BUTTON1_CLICKED)
{
// Single click - move cursor and clear selection
int fileRow, fileCol;
if (mouseToFilePos(event.y, event.x, fileRow, fileCol))
{
clearSelection();
updateCursorAndViewport(fileRow, fileCol);
}
}
else if (event.bstate & BUTTON4_PRESSED)
{
// Scroll up
scrollUp();
}
else if (event.bstate & BUTTON5_PRESSED)
{
// Scroll down
scrollDown();
}
}
void Editor::clearSelection()
{
hasSelection = false;
isSelecting = false;
selectionStartLine = 0;
selectionStartCol = 0;
selectionEndLine = 0;
selectionEndCol = 0;
}
void Editor::moveCursorUp()
{
if (cursorLine > 0)
{
cursorLine--;
if (cursorLine < viewportTop)
{
viewportTop = cursorLine;
}
if (cursorCol > 0)
{
std::string line = buffer.getLine(cursorLine);
int lineLen = static_cast<int>(line.length());
if (cursorCol > lineLen)
{
std::string expandedLine = expandTabs(line, tabSize);
cursorCol =
std::min(cursorCol, static_cast<int>(expandedLine.length()));
}
}
}
// Note: Selection handling now done in InputHandler
}
void Editor::moveCursorDown()
{
int maxLine = buffer.getLineCount() - 1;
if (cursorLine < maxLine)
{
cursorLine++;
if (cursorLine >= viewportTop + viewportHeight)
{
viewportTop = cursorLine - viewportHeight + 1;
}
if (cursorCol > 0)
{
std::string line = buffer.getLine(cursorLine);
int lineLen = static_cast<int>(line.length());
if (cursorCol > lineLen)
{
std::string expandedLine = expandTabs(line, tabSize);
cursorCol =
std::min(cursorCol, static_cast<int>(expandedLine.length()));
}
}
}
}
void Editor::moveCursorLeft()
{
if (cursorCol > 0)
{
cursorCol--;
if (cursorCol < viewportLeft)
{
viewportLeft = cursorCol;
}
}
else if (cursorLine > 0)
{
cursorLine--;
int currentTabSize = ConfigManager::getTabSize();
std::string expandedLine =
expandTabs(buffer.getLine(cursorLine), currentTabSize);
cursorCol = expandedLine.length();
if (cursorLine < viewportTop)
{
viewportTop = cursorLine;
}
int rows, cols;
getmaxyx(stdscr, rows, cols);
bool show_line_numbers = ConfigManager::getLineNumbers();
int lineNumWidth =
show_line_numbers ? std::to_string(buffer.getLineCount()).length() : 0;
int contentWidth = cols - (show_line_numbers ? (lineNumWidth + 3) : 0);
if (contentWidth > 0 && cursorCol >= viewportLeft + contentWidth)
{
viewportLeft = cursorCol - contentWidth + 1;
if (viewportLeft < 0)
viewportLeft = 0;
}
}
}
void Editor::moveCursorRight()
{
std::string line = buffer.getLine(cursorLine);
if (cursorCol < static_cast<int>(line.length()))
{
if (line[cursorCol] != '\t')
{
cursorCol++;
}
else
{
int currentTabSize = ConfigManager::getTabSize();
std::string expandedLine = expandTabs(line, currentTabSize);
if (cursorCol < static_cast<int>(expandedLine.length()))
{
cursorCol++;
}
}
int rows, cols;
getmaxyx(stdscr, rows, cols);
bool show_line_numbers = ConfigManager::getLineNumbers();
int lineNumWidth =
show_line_numbers ? std::to_string(buffer.getLineCount()).length() : 0;
int contentWidth = cols - (show_line_numbers ? (lineNumWidth + 3) : 0);
if (contentWidth > 0 && cursorCol >= viewportLeft + contentWidth)
{
viewportLeft = cursorCol - contentWidth + 1;
}
}
else if (cursorLine < buffer.getLineCount() - 1)
{
cursorLine++;
cursorCol = 0;
if (cursorLine >= viewportTop + viewportHeight)
{
viewportTop = cursorLine - viewportHeight + 1;
}
viewportLeft = 0;
}
}
void Editor::pageUp()
{
for (int i = 0; i < 10; i++)
{
moveCursorUp();
}
}
void Editor::pageDown()
{
for (int i = 0; i < 10; i++)
{
moveCursorDown();
}
}
void Editor::moveCursorToLineStart()
{
cursorCol = 0;
if (cursorCol < viewportLeft)
{
viewportLeft = 0;
}
// Selection handling is done in InputHandler, not here
// This method just moves the cursor
}
void Editor::moveCursorToLineEnd()
{
int currentTabSize = ConfigManager::getTabSize();
std::string expandedLine =
expandTabs(buffer.getLine(cursorLine), currentTabSize);
cursorCol = static_cast<int>(expandedLine.length());
int rows, cols;
getmaxyx(stdscr, rows, cols);
bool show_line_numbers = ConfigManager::getLineNumbers();
int lineNumWidth =
show_line_numbers ? std::to_string(buffer.getLineCount()).length() : 0;
int contentWidth = cols - (show_line_numbers ? (lineNumWidth + 3) : 0);
if (contentWidth > 0 && cursorCol >= viewportLeft + contentWidth)
{
viewportLeft = cursorCol - contentWidth + 1;
if (viewportLeft < 0)
viewportLeft = 0;
}
// Selection handling is done in InputHandler, not here
// This method just moves the cursor
}
void Editor::scrollUp(int linesToScroll)
{
viewportTop -= linesToScroll;
if (viewportTop < 0)
viewportTop = 0;
if (cursorLine < viewportTop)
{
cursorLine = viewportTop;
if (cursorLine < 0)
cursorLine = 0;
if (cursorLine >= buffer.getLineCount())
{
cursorLine = buffer.getLineCount() - 1;
}
std::string expandedLine = expandTabs(buffer.getLine(cursorLine), tabSize);
cursorCol = std::min(cursorCol, static_cast<int>(expandedLine.length()));
}
}
void Editor::scrollDown(int linesToScroll)
{
int maxViewportTop = buffer.getLineCount() - viewportHeight;
if (maxViewportTop < 0)
maxViewportTop = 0;
viewportTop += linesToScroll;
if (viewportTop > maxViewportTop)
viewportTop = maxViewportTop;
if (viewportTop < 0)
viewportTop = 0;
if (cursorLine >= viewportTop + viewportHeight)
{
cursorLine = viewportTop + viewportHeight - 1;
int maxLine = buffer.getLineCount() - 1;
if (cursorLine > maxLine)
cursorLine = maxLine;
if (cursorLine < 0)
cursorLine = 0;
std::string expandedLine = expandTabs(buffer.getLine(cursorLine), tabSize);
cursorCol = std::min(cursorCol, static_cast<int>(expandedLine.length()));
}
}
void Editor::validateCursorAndViewport()
{
if (buffer.getLineCount() == 0)
return;
int maxLine = buffer.getLineCount() - 1;
if (cursorLine < 0)
cursorLine = 0;
if (cursorLine > maxLine)
cursorLine = maxLine;
std::string expandedLine = expandTabs(buffer.getLine(cursorLine), tabSize);
if (cursorCol < 0)
cursorCol = 0;
if (cursorCol > static_cast<int>(expandedLine.length()))
{
cursorCol = static_cast<int>(expandedLine.length());
}
int maxViewportTop = buffer.getLineCount() - viewportHeight;
if (maxViewportTop < 0)
maxViewportTop = 0;
if (viewportTop < 0)
viewportTop = 0;
if (viewportTop > maxViewportTop)
viewportTop = maxViewportTop;
if (viewportLeft < 0)
viewportLeft = 0;
if (cursorLine < viewportTop)
{
viewportTop = cursorLine;
}
else if (cursorLine >= viewportTop + viewportHeight)
{
viewportTop = cursorLine - viewportHeight + 1;
if (viewportTop < 0)
viewportTop = 0;
if (viewportTop > maxViewportTop)
viewportTop = maxViewportTop;
}
}
// =================================================================
// File Operations
// =================================================================
void Editor::debugPrintState(const std::string &context)
{
std::cerr << "=== EDITOR STATE DEBUG: " << context << " ===" << std::endl;
std::cerr << "cursorLine: " << cursorLine << std::endl;
std::cerr << "cursorCol: " << cursorCol << std::endl;
std::cerr << "viewportTop: " << viewportTop << std::endl;
std::cerr << "viewportLeft: " << viewportLeft << std::endl;
std::cerr << "buffer.getLineCount(): " << buffer.getLineCount() << std::endl;
std::cerr << "buffer.size(): " << buffer.size() << std::endl;
std::cerr << "isModified: " << isModified << std::endl;
// std::cerr << "currentMode: " << (int)currentMode << std::endl;
if (cursorLine < buffer.getLineCount())
{
std::string currentLine = buffer.getLine(cursorLine);
std::cerr << "currentLine length: " << currentLine.length() << std::endl;
std::cerr << "currentLine content: '" << currentLine << "'" << std::endl;
}
else
{
std::cerr << "ERROR: cursorLine out of bounds!" << std::endl;
}
std::cerr << "hasSelection: " << hasSelection << std::endl;
std::cerr << "isSelecting: " << isSelecting << std::endl;
std::cerr << "undoStack.size(): " << undoStack.size() << std::endl;
std::cerr << "redoStack.size(): " << redoStack.size() << std::endl;
std::cerr << "=== END DEBUG ===" << std::endl;
}
bool Editor::validateEditorState()
{
bool valid = true;
if (cursorLine < 0 || cursorLine >= buffer.getLineCount())
{
std::cerr << "INVALID: cursorLine out of bounds: " << cursorLine
<< " (max: " << buffer.getLineCount() - 1 << ")" << std::endl;
valid = false;
}
if (cursorCol < 0)
{
std::cerr << "INVALID: cursorCol negative: " << cursorCol << std::endl;
valid = false;
}
if (cursorLine >= 0 && cursorLine < buffer.getLineCount())
{
std::string line = buffer.getLine(cursorLine);
if (cursorCol > static_cast<int>(line.length()))
{
std::cerr << "INVALID: cursorCol past end of line: " << cursorCol
<< " (line length: " << line.length() << ")" << std::endl;
valid = false;
}
}
if (viewportTop < 0)
{
std::cerr << "INVALID: viewportTop negative: " << viewportTop << std::endl;
valid = false;
}
if (viewportLeft < 0)
{
std::cerr << "INVALID: viewportLeft negative: " << viewportLeft
<< std::endl;
valid = false;
}
return valid;
}
bool Editor::loadFile(const std::string &fname)
{
filename = fname;
if (syntaxHighlighter)
{
std::string extension = getFileExtension();
syntaxHighlighter->setLanguage(extension);
}
if (!buffer.loadFromFile(filename))
{
buffer.clear();
buffer.insertLine(0, "");
return false;
}
// Set language but DON'T parse yet - parsing happens on first display
isModified = false;
return true;
}
bool Editor::saveFile()
{
if (filename.empty())
{
return false;
}
// Set flag to prevent saveState() during file operations
isSaving = true;
bool success = buffer.saveToFile(filename);
if (success)
{
isModified = false;
}
isSaving = false; // Reset flag
return success;
}
// =================================================================
// Text Editing Operations
// =================================================================
void Editor::insertChar(char ch)
{
if (cursorLine < 0 || cursorLine >= buffer.getLineCount())
return;
if (useDeltaUndo_ && !isUndoRedoing)
{
EditDelta delta = createDeltaForInsertChar(ch);
std::string line = buffer.getLine(cursorLine);
if (cursorCol > static_cast<int>(line.length()))
cursorCol = line.length();
if (cursorCol < 0)
cursorCol = 0;
size_t byte_pos = buffer.lineColToPos(cursorLine, cursorCol);
line.insert(cursorCol, 1, ch);
buffer.replaceLine(cursorLine, line);
cursorCol++;
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(
buffer, byte_pos, 0, 1, cursorLine, cursorCol - 1, cursorLine,
cursorCol - 1, cursorLine, cursorCol);
// NEW: Always invalidate cache after edit
syntaxHighlighter->invalidateLineCache(cursorLine);
}
// Update viewport
int rows, cols;
getmaxyx(stdscr, rows, cols);
bool show_line_numbers = ConfigManager::getLineNumbers();
int lineNumWidth =
show_line_numbers ? std::to_string(buffer.getLineCount()).length() : 0;
int contentWidth = cols - (show_line_numbers ? (lineNumWidth + 3) : 0);
if (contentWidth > 0 && cursorCol >= viewportLeft + contentWidth)
{
viewportLeft = cursorCol - contentWidth + 1;
}
// Complete delta
delta.postCursorLine = cursorLine;
delta.postCursorCol = cursorCol;
delta.postViewportTop = viewportTop;
delta.postViewportLeft = viewportLeft;
addDelta(delta);
// FIX: Auto-commit on timeout OR boundary characters for immediate
// highlighting
auto now = std::chrono::steady_clock::now();
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
now - currentDeltaGroup_.timestamp)
.count();
// Boundary characters that should trigger immediate commit
bool is_boundary_char =
(ch == '>' || ch == ')' || ch == '}' || ch == ']' || ch == ';' ||
ch == ',' || ch == ' ' || ch == '\t' || ch == '<' || ch == '(' ||
ch == '{' || ch == '[');
if (elapsed > UNDO_GROUP_TIMEOUT_MS || is_boundary_char)
{
commitDeltaGroup();
beginDeltaGroup();
}
markModified();
}
else if (!isUndoRedoing)
{
// OLD: Full-state undo (fallback)
saveState();
std::string line = buffer.getLine(cursorLine);
if (cursorCol > static_cast<int>(line.length()))
cursorCol = line.length();
if (cursorCol < 0)
cursorCol = 0;
size_t byte_pos = buffer.lineColToPos(cursorLine, cursorCol);
line.insert(cursorCol, 1, ch);
buffer.replaceLine(cursorLine, line);
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(buffer, byte_pos, 0, 1, cursorLine,
cursorCol, cursorLine, cursorCol,
cursorLine, cursorCol + 1);
syntaxHighlighter->invalidateLineRange(cursorLine, cursorLine);
}
cursorCol++;
markModified();
int rows, cols;
getmaxyx(stdscr, rows, cols);
int lineNumWidth = std::to_string(buffer.getLineCount()).length();
int contentWidth = cols - lineNumWidth - 3;
if (contentWidth > 0 && cursorCol >= viewportLeft + contentWidth)
{
viewportLeft = cursorCol - contentWidth + 1;
}
}
}
void Editor::insertNewline()
{
if (useDeltaUndo_ && !isUndoRedoing)
{
EditorSnapshot before = captureSnapshot();
EditDelta delta = createDeltaForNewline();
size_t byte_pos = buffer.lineColToPos(cursorLine, cursorCol);
// 1. MODIFY BUFFER FIRST
splitLineAtCursor();
cursorLine++;
cursorCol = 0;
// 2. THEN notify Tree-sitter AFTER buffer change
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(
buffer, byte_pos, 0, 1, // Inserted 1 byte (newline)
delta.preCursorLine, delta.preCursorCol, // OLD position
delta.preCursorLine, delta.preCursorCol, cursorLine,
0); // NEW position
// Invalidate from split point onwards
syntaxHighlighter->invalidateLineRange(cursorLine - 1,
buffer.getLineCount() - 1);
}
if (cursorLine >= viewportTop + viewportHeight)
{
viewportTop = cursorLine - viewportHeight + 1;
}
viewportLeft = 0;
// Complete delta
delta.postCursorLine = cursorLine;
delta.postCursorCol = cursorCol;
delta.postViewportTop = viewportTop;
delta.postViewportLeft = viewportLeft;
ValidationResult valid = validateState("After insertNewline");
if (valid)
{
addDelta(delta);
// Newlines always commit the current group
commitDeltaGroup();
beginDeltaGroup();
}
else
{
std::cerr << "VALIDATION FAILED in insertNewline\n";
std::cerr << valid.error << "\n";
}
markModified();
}
else if (!isUndoRedoing)
{
// OLD: Full-state undo
saveState();
size_t byte_pos = buffer.lineColToPos(cursorLine, cursorCol);
splitLineAtCursor();
cursorLine++;
cursorCol = 0;
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(buffer, byte_pos, 0, 1,
cursorLine - 1, 0, cursorLine - 1,
0, cursorLine, 0);
syntaxHighlighter->invalidateLineRange(cursorLine - 1,
buffer.getLineCount() - 1);
}
if (cursorLine >= viewportTop + viewportHeight)
{
viewportTop = cursorLine - viewportHeight + 1;
}
viewportLeft = 0;
markModified();
}
}
void Editor::deleteChar()
{
if (useDeltaUndo_ && !isUndoRedoing)
{
EditorSnapshot before = captureSnapshot();
EditDelta delta = createDeltaForDeleteChar();
std::string line = buffer.getLine(cursorLine);
if (cursorCol < static_cast<int>(line.length()))
{
size_t byte_pos = buffer.lineColToPos(cursorLine, cursorCol);
line.erase(cursorCol, 1);
buffer.replaceLine(cursorLine, line);
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(
buffer, byte_pos, 1, 0, cursorLine, cursorCol, cursorLine,
cursorCol + 1, cursorLine, cursorCol);
// NEW: Always invalidate cache after edit
syntaxHighlighter->invalidateLineCache(cursorLine);
}
}
else if (cursorLine < buffer.getLineCount() - 1)
{
size_t byte_pos = buffer.lineColToPos(cursorLine, line.length());
std::string nextLine = buffer.getLine(cursorLine + 1);
buffer.replaceLine(cursorLine, line + nextLine);
buffer.deleteLine(cursorLine + 1);
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(
buffer, byte_pos, 1, 0, cursorLine, (uint32_t)line.length(),
cursorLine + 1, 0, cursorLine, (uint32_t)line.length());
syntaxHighlighter->invalidateLineRange(cursorLine,
buffer.getLineCount() - 1);
}
}
delta.postCursorLine = cursorLine;
delta.postCursorCol = cursorCol;
delta.postViewportTop = viewportTop;
delta.postViewportLeft = viewportLeft;
ValidationResult valid = validateState("After deleteChar");
if (valid)
{
addDelta(delta);
if (delta.operation == EditDelta::JOIN_LINES)
{
commitDeltaGroup();
beginDeltaGroup();
}
}
else
{
std::cerr << "VALIDATION FAILED in deleteChar\n";
std::cerr << valid.error << "\n";
}
markModified();
}
else if (!isUndoRedoing)
{
saveState();
std::string line = buffer.getLine(cursorLine);
if (cursorCol < static_cast<int>(line.length()))
{
size_t byte_pos = buffer.lineColToPos(cursorLine, cursorCol);
line.erase(cursorCol, 1);
buffer.replaceLine(cursorLine, line);
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(
buffer, byte_pos, 1, 0, cursorLine, cursorCol, cursorLine,
cursorCol + 1, cursorLine, cursorCol);
// NEW: Always invalidate cache after edit
syntaxHighlighter->invalidateLineCache(cursorLine);
}
markModified();
}
else if (cursorLine < buffer.getLineCount() - 1)
{
size_t byte_pos = buffer.lineColToPos(cursorLine, line.length());
std::string nextLine = buffer.getLine(cursorLine + 1);
buffer.replaceLine(cursorLine, line + nextLine);
buffer.deleteLine(cursorLine + 1);
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(
buffer, byte_pos, 1, 0, cursorLine, (uint32_t)line.length(),
cursorLine + 1, 0, cursorLine, (uint32_t)line.length());
syntaxHighlighter->invalidateLineRange(cursorLine,
buffer.getLineCount() - 1);
}
markModified();
}
}
}
void Editor::backspace()
{
if (useDeltaUndo_ && !isUndoRedoing)
{
EditorSnapshot before = captureSnapshot();
EditDelta delta = createDeltaForBackspace();
if (cursorCol > 0)
{
std::string line = buffer.getLine(cursorLine);
size_t byte_pos = buffer.lineColToPos(cursorLine, cursorCol - 1);
line.erase(cursorCol - 1, 1);
buffer.replaceLine(cursorLine, line);
cursorCol--;
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(
buffer, byte_pos, 1, 0, cursorLine, cursorCol, cursorLine,
cursorCol + 1, cursorLine, cursorCol);
// NEW: Always invalidate cache after edit
syntaxHighlighter->invalidateLineCache(cursorLine);
}
if (cursorCol < viewportLeft)
{
viewportLeft = cursorCol;
}
}
else if (cursorLine > 0)
{
std::string currentLine = buffer.getLine(cursorLine);
std::string prevLine = buffer.getLine(cursorLine - 1);
size_t byte_pos = buffer.lineColToPos(cursorLine - 1, prevLine.length());
int oldCursorLine = cursorLine;
cursorCol = static_cast<int>(prevLine.length());
cursorLine--;
buffer.replaceLine(cursorLine, prevLine + currentLine);
buffer.deleteLine(cursorLine + 1);
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(
buffer, byte_pos, 1, 0, cursorLine, cursorCol, oldCursorLine, 0,
cursorLine, cursorCol);
syntaxHighlighter->invalidateLineRange(cursorLine,
buffer.getLineCount() - 1);
}
}
delta.postCursorLine = cursorLine;
delta.postCursorCol = cursorCol;
delta.postViewportTop = viewportTop;
delta.postViewportLeft = viewportLeft;
ValidationResult valid = validateState("After backspace");
if (valid)
{
addDelta(delta);
if (delta.operation == EditDelta::JOIN_LINES)
{
commitDeltaGroup();
beginDeltaGroup();
}
}
else
{
std::cerr << "VALIDATION FAILED in backspace\n";
std::cerr << valid.error << "\n";
}
markModified();
}
else if (!isUndoRedoing)
{
saveState();
if (cursorCol > 0)
{
size_t byte_pos = buffer.lineColToPos(cursorLine, cursorCol - 1);
std::string line = buffer.getLine(cursorLine);
line.erase(cursorCol - 1, 1);
buffer.replaceLine(cursorLine, line);
cursorCol--;
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(
buffer, byte_pos, 1, 0, cursorLine, cursorCol, cursorLine,
cursorCol + 1, cursorLine, cursorCol);
// NEW: Always invalidate cache after edit
syntaxHighlighter->invalidateLineCache(cursorLine);
}
if (cursorCol < viewportLeft)
{
viewportLeft = cursorCol;
}
markModified();
}
else if (cursorLine > 0)
{
std::string currentLine = buffer.getLine(cursorLine);
std::string prevLine = buffer.getLine(cursorLine - 1);
size_t byte_pos = buffer.lineColToPos(cursorLine - 1, prevLine.length());
cursorCol = static_cast<int>(prevLine.length());
cursorLine--;
buffer.replaceLine(cursorLine, prevLine + currentLine);
buffer.deleteLine(cursorLine + 1);
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(
buffer, byte_pos, 1, 0, cursorLine, cursorCol, cursorLine + 1, 0,
cursorLine, cursorCol);
syntaxHighlighter->invalidateLineRange(cursorLine,
buffer.getLineCount() - 1);
}
markModified();
}
}
}
void Editor::deleteLine()
{
// SAVE STATE BEFORE MODIFICATION
if (!isUndoRedoing)
{
saveState();
}
if (buffer.getLineCount() == 1)
{
std::string line = buffer.getLine(0);
size_t byte_pos = 0;
buffer.replaceLine(0, "");
cursorCol = 0;
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(buffer, byte_pos, line.length(), 0,
0, 0, 0, (uint32_t)line.length(),
0, 0);
syntaxHighlighter->invalidateLineRange(0, 0);
}
// buffer.replaceLine(0, "");
// cursorCol = 0;
}
else
{
size_t byte_pos = buffer.lineColToPos(cursorLine, 0);
std::string line = buffer.getLine(cursorLine);
size_t line_length = line.length();
bool has_newline = (cursorLine < buffer.getLineCount() - 1);
size_t delete_bytes = line_length + (has_newline ? 1 : 0);
buffer.deleteLine(cursorLine);
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(
buffer, byte_pos, delete_bytes, 0, cursorLine, 0,
cursorLine + (has_newline ? 1 : 0),
has_newline ? 0 : (uint32_t)line_length, cursorLine, 0);
syntaxHighlighter->invalidateLineRange(cursorLine,
buffer.getLineCount() - 1);
}
if (cursorLine >= buffer.getLineCount())
{
cursorLine = buffer.getLineCount() - 1;
}
line = buffer.getLine(cursorLine);
if (cursorCol > static_cast<int>(line.length()))
{
cursorCol = static_cast<int>(line.length());
}
}
validateCursorAndViewport();
markModified();
}
// =================================================================
// Undo/Redo System
// =================================================================
void Editor::deleteSelection()
{
if (!hasSelection && !isSelecting)
return;
if (useDeltaUndo_ && !isUndoRedoing)
{
EditorSnapshot before = captureSnapshot();
EditDelta delta = createDeltaForDeleteSelection();
auto selection = getNormalizedSelection();
int startLine = selection.first.first;
int startCol = selection.first.second;
int endLine = selection.second.first;
int endCol = selection.second.second;
size_t start_byte = buffer.lineColToPos(startLine, startCol);
size_t end_byte = buffer.lineColToPos(endLine, endCol);
size_t delete_bytes = end_byte - start_byte;
// 1. MODIFY BUFFER FIRST
if (startLine == endLine)
{
std::string line = buffer.getLine(startLine);
line.erase(startCol, endCol - startCol);
buffer.replaceLine(startLine, line);
}
else
{
std::string firstLine = buffer.getLine(startLine);
std::string lastLine = buffer.getLine(endLine);
std::string newLine =
firstLine.substr(0, startCol) + lastLine.substr(endCol);
buffer.replaceLine(startLine, newLine);
for (int i = endLine; i > startLine; i--)
{
buffer.deleteLine(i);
}
}
// 2. THEN notify Tree-sitter AFTER buffer change
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(
buffer, start_byte, delete_bytes, 0, // Deleted bytes
startLine, startCol, endLine, endCol, startLine, startCol);
syntaxHighlighter->invalidateLineRange(startLine,
buffer.getLineCount() - 1);
}
updateCursorAndViewport(startLine, startCol);
clearSelection();
// Complete delta
delta.postCursorLine = cursorLine;
delta.postCursorCol = cursorCol;
delta.postViewportTop = viewportTop;
delta.postViewportLeft = viewportLeft;
ValidationResult valid = validateState("After deleteSelection");
if (valid)
{
addDelta(delta);
commitDeltaGroup();
beginDeltaGroup();
}
else
{
std::cerr << "VALIDATION FAILED in deleteSelection\n";
std::cerr << valid.error << "\n";
}
markModified();
}
else if (!isUndoRedoing)
{
// OLD: Full-state undo
saveState();
auto selection = getNormalizedSelection();
int startLine = selection.first.first;
int startCol = selection.first.second;
int endLine = selection.second.first;
int endCol = selection.second.second;
size_t start_byte = buffer.lineColToPos(startLine, startCol);
size_t end_byte = buffer.lineColToPos(endLine, endCol);
size_t delete_bytes = end_byte - start_byte;
if (startLine == endLine)
{
std::string line = buffer.getLine(startLine);
line.erase(startCol, endCol - startCol);
buffer.replaceLine(startLine, line);
}
else
{
std::string firstLine = buffer.getLine(startLine);
std::string lastLine = buffer.getLine(endLine);
std::string newLine =
firstLine.substr(0, startCol) + lastLine.substr(endCol);
buffer.replaceLine(startLine, newLine);
for (int i = endLine; i > startLine; i--)
{
buffer.deleteLine(i);
}
}
if (syntaxHighlighter && !isUndoRedoing)
{
syntaxHighlighter->updateTreeAfterEdit(buffer, start_byte, delete_bytes,
0, startLine, startCol, endLine,
endCol, startLine, startCol);
syntaxHighlighter->invalidateLineRange(startLine,
buffer.getLineCount() - 1);
}
updateCursorAndViewport(startLine, startCol);
clearSelection();
markModified();
}
}
void Editor::undo()
{
if (useDeltaUndo_)
{
// Commit any pending delta group first
if (!currentDeltaGroup_.isEmpty())
{
commitDeltaGroup();
}
if (deltaUndoStack_.empty())
{
return;
}
#ifdef DEBUG_DELTA_UNDO
std::cerr << "\n=== UNDO START ===\n";
EditorSnapshot beforeUndo = captureSnapshot();
#endif
// Get the delta group to undo
DeltaGroup group = deltaUndoStack_.top();
deltaUndoStack_.pop();
#ifdef DEBUG_DELTA_UNDO
std::cerr << "Undoing group:\n" << group.toString() << "\n";
#endif
// Track affected line range for incremental highlighting
int minAffectedLine = buffer.getLineCount();
int maxAffectedLine = 0;
// Apply deltas in REVERSE order
for (auto it = group.deltas.rbegin(); it != group.deltas.rend(); ++it)
{
// Track which lines are affected
minAffectedLine =
std::min(minAffectedLine, std::min(it->startLine, it->preCursorLine));
maxAffectedLine =
std::max(maxAffectedLine, std::max(it->endLine, it->postCursorLine));
applyDeltaReverse(*it);
#ifdef DEBUG_DELTA_UNDO
ValidationResult valid = validateState("After undo delta");
if (!valid)
{
std::cerr << "CRITICAL: Validation failed during undo!\n";
std::cerr << valid.error << "\n";
}
#endif
}
// Save to redo stack
deltaRedoStack_.push(group);
// FIXED: Incremental syntax update instead of full reparse
if (syntaxHighlighter)
{
// Only invalidate affected line range, not entire cache
syntaxHighlighter->invalidateLineRange(minAffectedLine,
buffer.getLineCount() - 1);
// Use viewport-only parsing for immediate visual update
syntaxHighlighter->parseViewportOnly(buffer, viewportTop);
// Schedule background full reparse (non-blocking)
syntaxHighlighter->scheduleBackgroundParse(buffer);
}
isModified = true;
#ifdef DEBUG_DELTA_UNDO
EditorSnapshot afterUndo = captureSnapshot();
std::cerr << "Affected lines: " << minAffectedLine << " to "
<< maxAffectedLine << "\n";
std::cerr << "=== UNDO END ===\n\n";
#endif
}
else
{
// OLD: Full-state undo (fallback)
if (undoStack.empty())
return;
isUndoRedoing = true;
redoStack.push(getCurrentState());
EditorState state = undoStack.top();
undoStack.pop();
restoreState(state);
if (syntaxHighlighter)
{
syntaxHighlighter->bufferChanged(buffer);
}
isModified = true;
isUndoRedoing = false;
}
}
void Editor::redo()
{
if (useDeltaUndo_)
{
if (deltaRedoStack_.empty())
{
return;
}
#ifdef DEBUG_DELTA_UNDO
std::cerr << "\n=== REDO START ===\n";
EditorSnapshot beforeRedo = captureSnapshot();
#endif
// Get the delta group to redo
DeltaGroup group = deltaRedoStack_.top();
deltaRedoStack_.pop();
#ifdef DEBUG_DELTA_UNDO
std::cerr << "Redoing group:\n" << group.toString() << "\n";
#endif
// Track affected line range
int minAffectedLine = buffer.getLineCount();
int maxAffectedLine = 0;
// Apply deltas in FORWARD order
for (const auto &delta : group.deltas)
{
minAffectedLine = std::min(
minAffectedLine, std::min(delta.startLine, delta.preCursorLine));
maxAffectedLine = std::max(maxAffectedLine,
std::max(delta.endLine, delta.postCursorLine));
applyDeltaForward(delta);
#ifdef DEBUG_DELTA_UNDO
ValidationResult valid = validateState("After redo delta");
if (!valid)
{
std::cerr << "CRITICAL: Validation failed during redo!\n";
std::cerr << valid.error << "\n";
}
#endif
}
// Save to undo stack
deltaUndoStack_.push(group);
// FIXED: Incremental syntax update
if (syntaxHighlighter)
{
syntaxHighlighter->invalidateLineRange(minAffectedLine,
buffer.getLineCount() - 1);
syntaxHighlighter->parseViewportOnly(buffer, viewportTop);
syntaxHighlighter->scheduleBackgroundParse(buffer);
}
isModified = true;
#ifdef DEBUG_DELTA_UNDO
EditorSnapshot afterRedo = captureSnapshot();
std::cerr << "Affected lines: " << minAffectedLine << " to "
<< maxAffectedLine << "\n";
std::cerr << "=== REDO END ===\n\n";
#endif
}
else
{
// OLD: Full-state redo (fallback)
if (redoStack.empty())
return;
isUndoRedoing = true;
undoStack.push(getCurrentState());
EditorState state = redoStack.top();
redoStack.pop();
restoreState(state);
if (syntaxHighlighter)
{
syntaxHighlighter->bufferChanged(buffer);
}
isModified = true;
isUndoRedoing = false;
}
}
EditorState Editor::getCurrentState()
{
EditorState state;
// Your existing serialization code
std::ostringstream oss;
for (int i = 0; i < buffer.getLineCount(); i++)
{
oss << buffer.getLine(i);
if (i < buffer.getLineCount() - 1)
{
oss << "\n";
}
}
state.content = oss.str();
// Save cursor/viewport state regardless
state.cursorLine = cursorLine;
state.cursorCol = cursorCol;
state.viewportTop = viewportTop;
state.viewportLeft = viewportLeft;
return state;
}
void Editor::restoreState(const EditorState &state)
{
// Clear buffer and reload content
buffer.clear();
std::istringstream iss(state.content);
std::string line;
int lineNum = 0;
while (std::getline(iss, line))
{
buffer.insertLine(lineNum++, line);
}
// If no lines were added, add empty line
if (buffer.getLineCount() == 0)
{
buffer.insertLine(0, "");
}
// Restore cursor and viewport
cursorLine = state.cursorLine;
cursorCol = state.cursorCol;
viewportTop = state.viewportTop;
viewportLeft = state.viewportLeft;
validateCursorAndViewport();
}
void Editor::limitUndoStack()
{
while (undoStack.size() > MAX_UNDO_LEVELS)
{
// Remove oldest state (bottom of stack)
std::stack<EditorState> temp;
bool first = true;
while (!undoStack.empty())
{
if (first)
{
first = false;
undoStack.pop(); // Skip the oldest
}
else
{
temp.push(undoStack.top());
undoStack.pop();
}
}
// Restore stack in correct order
while (!temp.empty())
{
undoStack.push(temp.top());
temp.pop();
}
}
}
// =================================================================
// Internal Helpers
// =================================================================
void Editor::markModified() { isModified = true; }
void Editor::splitLineAtCursor()
{
std::string line = buffer.getLine(cursorLine);
std::string leftPart = line.substr(0, cursorCol);
std::string rightPart = line.substr(cursorCol);
buffer.replaceLine(cursorLine, leftPart);
buffer.insertLine(cursorLine + 1, rightPart);
}
void Editor::joinLineWithNext()
{
if (cursorLine < buffer.getLineCount() - 1)
{
std::string currentLine = buffer.getLine(cursorLine);
std::string nextLine = buffer.getLine(cursorLine + 1);
buffer.replaceLine(cursorLine, currentLine + nextLine);
buffer.deleteLine(cursorLine + 1);
}
}
std::pair<std::pair<int, int>, std::pair<int, int>>
Editor::getNormalizedSelection()
{
int startLine = selectionStartLine;
int startCol = selectionStartCol;
int endLine = selectionEndLine;
int endCol = selectionEndCol;
// Always normalize so start < end
if (startLine > endLine || (startLine == endLine && startCol > endCol))
{
std::swap(startLine, endLine);
std::swap(startCol, endCol);
}
return {{startLine, startCol}, {endLine, endCol}};
}
std::string Editor::getSelectedText()
{
if (!hasSelection && !isSelecting)
return "";
auto [start, end] = getNormalizedSelection();
int startLine = start.first, startCol = start.second;
int endLine = end.first, endCol = end.second;
std::ostringstream result;
if (startLine == endLine)
{
// Single line selection
std::string line = buffer.getLine(startLine);
result << line.substr(startCol, endCol - startCol);
}
else
{
// Multi-line selection
for (int i = startLine; i <= endLine; i++)
{
std::string line = buffer.getLine(i);
if (i == startLine)
{
result << line.substr(startCol);
}
else if (i == endLine)
{
result << line.substr(0, endCol);
}
else
{
result << line;
}
if (i < endLine)
{
result << "\n";
}
}
}
// CRITICAL FIX: Actually return the result!
return result.str();
}
// Selection management
void Editor::startSelectionIfNeeded()
{
if (!hasSelection && !isSelecting)
{
isSelecting = true;
selectionStartLine = cursorLine;
selectionStartCol = cursorCol;
selectionEndLine = cursorLine;
selectionEndCol = cursorCol;
}
}
void Editor::updateSelectionEnd()
{
if (isSelecting || hasSelection)
{
selectionEndLine = cursorLine;
selectionEndCol = cursorCol;
hasSelection = true;
}
}
// Clipboard operations
void Editor::copySelection()
{
if (!hasSelection && !isSelecting)
return;
clipboard = getSelectedText();
// On Unix, also copy to system clipboard using xclip or xsel
#ifndef _WIN32
FILE *pipe = popen("xclip -selection clipboard 2>/dev/null || xsel "
"--clipboard --input 2>/dev/null",
"w");
if (pipe)
{
fwrite(clipboard.c_str(), 1, clipboard.length(), pipe);
pclose(pipe);
}
#endif
}
void Editor::cutSelection()
{
if (!hasSelection && !isSelecting)
return;
copySelection();
deleteSelection();
}
void Editor::pasteFromClipboard()
{
// Try to get from system clipboard first
#ifndef _WIN32
FILE *pipe = popen("xclip -selection clipboard -o 2>/dev/null || xsel "
"--clipboard --output 2>/dev/null",
"r");
if (pipe)
{
char buffer_chars[4096];
std::string result;
while (fgets(buffer_chars, sizeof(buffer_chars), pipe))
{
result += buffer_chars;
}
pclose(pipe);
if (!result.empty())
{
clipboard = result;
}
}
#endif
if (clipboard.empty())
return;
// SAVE STATE BEFORE PASTE (single undo point for entire paste)
if (!isUndoRedoing)
{
saveState();
}
// Delete selection if any
if (hasSelection || isSelecting)
{
deleteSelection();
}
// Insert clipboard content character by character
// Note: Each insertChar/insertNewline will NOT call saveState
// because we already saved it above
for (char ch : clipboard)
{
if (ch == '\n')
{
insertNewline();
}
else
{
insertChar(ch);
}
}
}
void Editor::selectAll()
{
if (buffer.getLineCount() == 0)
return;
selectionStartLine = 0;
selectionStartCol = 0;
selectionEndLine = buffer.getLineCount() - 1;
std::string lastLine = buffer.getLine(selectionEndLine);
selectionEndCol = static_cast<int>(lastLine.length());
hasSelection = true;
isSelecting = false;
}
void Editor::initializeViewportHighlighting()
{
if (syntaxHighlighter)
{
// Pre-parse viewport so first display() is instant
syntaxHighlighter->parseViewportOnly(buffer, viewportTop);
}
}
// Cursor
void Editor::setCursorMode()
{
switch (currentMode)
{
case CursorMode::NORMAL:
// Block cursor (solid block)
printf("\033[2 q");
fflush(stdout);
break;
case CursorMode::INSERT:
// Vertical bar cursor (thin lifne like VSCode/modern editors)
printf("\033[6 q");
fflush(stdout);
break;
case CursorMode::VISUAL:
// Underline cursor for visual mode
printf("\033[4 q");
fflush(stdout);
break;
default:
printf("\033[6 q");
fflush(stdout);
break;
}
}
// === Delta Group Management ===
void Editor::beginDeltaGroup()
{
currentDeltaGroup_ = DeltaGroup();
currentDeltaGroup_.initialLineCount = buffer.getLineCount();
currentDeltaGroup_.initialBufferSize = buffer.size();
currentDeltaGroup_.timestamp = std::chrono::steady_clock::now();
}
void Editor::addDelta(const EditDelta &delta)
{
currentDeltaGroup_.addDelta(delta);
// DEBUG: Validate after every delta in debug builds
#ifdef DEBUG_DELTA_UNDO
ValidationResult valid = validateState("After adding delta");
if (!valid)
{
std::cerr << "VALIDATION FAILED after delta:\n";
std::cerr << delta.toString() << "\n";
std::cerr << "Error: " << valid.error << "\n";
std::cerr << "Current state: " << captureSnapshot().toString() << "\n";
}
#endif
}
void Editor::commitDeltaGroup()
{
if (currentDeltaGroup_.isEmpty())
{
return;
}
// Validate before committing
ValidationResult valid = validateState("Before committing delta group");
if (!valid)
{
std::cerr << "WARNING: Invalid state before commit, discarding group\n";
std::cerr << valid.error << "\n";
currentDeltaGroup_ = DeltaGroup();
return;
}
deltaUndoStack_.push(currentDeltaGroup_);
// Clear redo stack on new edit
while (!deltaRedoStack_.empty())
{
deltaRedoStack_.pop();
}
// Limit stack size
while (deltaUndoStack_.size() > MAX_UNDO_LEVELS)
{
// Remove oldest (bottom of stack)
std::stack<DeltaGroup> temp;
bool first = true;
while (!deltaUndoStack_.empty())
{
if (first)
{
first = false;
deltaUndoStack_.pop(); // Discard oldest
}
else
{
temp.push(deltaUndoStack_.top());
deltaUndoStack_.pop();
}
}
while (!temp.empty())
{
deltaUndoStack_.push(temp.top());
temp.pop();
}
}
currentDeltaGroup_ = DeltaGroup();
}
// === Delta Creation for Each Operation ===
EditDelta Editor::createDeltaForInsertChar(char ch)
{
EditDelta delta;
delta.operation = EditDelta::INSERT_CHAR;
// Capture state BEFORE edit
delta.preCursorLine = cursorLine;
delta.preCursorCol = cursorCol;
delta.preViewportTop = viewportTop;
delta.preViewportLeft = viewportLeft;
delta.startLine = cursorLine;
delta.startCol = cursorCol;
delta.endLine = cursorLine;
delta.endCol = cursorCol;
// Content: what we're inserting
delta.insertedContent = std::string(1, ch);
delta.deletedContent = ""; // Nothing deleted
// No structural change
delta.lineCountDelta = 0;
// Post-state will be filled after edit completes
return delta;
}
EditDelta Editor::createDeltaForDeleteChar()
{
EditDelta delta;
delta.operation = EditDelta::DELETE_CHAR;
// Capture state BEFORE deletion
delta.preCursorLine = cursorLine;
delta.preCursorCol = cursorCol;
delta.preViewportTop = viewportTop;
delta.preViewportLeft = viewportLeft;
delta.startLine = cursorLine;
delta.startCol = cursorCol;
// Capture what we're about to delete
std::string line = buffer.getLine(cursorLine);
if (cursorCol < static_cast<int>(line.length()))
{
// Deleting a character on current line
delta.deletedContent = std::string(1, line[cursorCol]);
delta.endLine = cursorLine;
delta.endCol = cursorCol + 1;
delta.lineCountDelta = 0;
}
else if (cursorLine < buffer.getLineCount() - 1)
{
// Deleting newline - will join lines
delta.operation = EditDelta::JOIN_LINES;
delta.deletedContent = "\n";
delta.endLine = cursorLine + 1;
delta.endCol = 0;
delta.lineCountDelta = -1;
// Save line contents for reversal
delta.firstLineBeforeJoin = line;
delta.secondLineBeforeJoin = buffer.getLine(cursorLine + 1);
}
delta.insertedContent = ""; // Nothing inserted
return delta;
}
EditDelta Editor::createDeltaForBackspace()
{
EditDelta delta;
delta.operation = EditDelta::DELETE_CHAR;
// Capture state BEFORE deletion
delta.preCursorLine = cursorLine;
delta.preCursorCol = cursorCol;
delta.preViewportTop = viewportTop;
delta.preViewportLeft = viewportLeft;
if (cursorCol > 0)
{
// Deleting character before cursor on same line
std::string line = buffer.getLine(cursorLine);
delta.deletedContent = std::string(1, line[cursorCol - 1]);
delta.startLine = cursorLine;
delta.startCol = cursorCol - 1;
delta.endLine = cursorLine;
delta.endCol = cursorCol;
delta.lineCountDelta = 0;
}
else if (cursorLine > 0)
{
// Backspace at line start - join with previous line
delta.operation = EditDelta::JOIN_LINES;
delta.deletedContent = "\n";
std::string prevLine = buffer.getLine(cursorLine - 1);
std::string currLine = buffer.getLine(cursorLine);
delta.startLine = cursorLine - 1;
delta.startCol = prevLine.length();
delta.endLine = cursorLine;
delta.endCol = 0;
delta.lineCountDelta = -1;
// Save line contents for reversal
delta.firstLineBeforeJoin = prevLine;
delta.secondLineBeforeJoin = currLine;
}
delta.insertedContent = ""; // Nothing inserted
return delta;
}
EditDelta Editor::createDeltaForNewline()
{
EditDelta delta;
delta.operation = EditDelta::SPLIT_LINE;
// Capture state BEFORE split
delta.preCursorLine = cursorLine;
delta.preCursorCol = cursorCol;
delta.preViewportTop = viewportTop;
delta.preViewportLeft = viewportLeft;
delta.startLine = cursorLine;
delta.startCol = cursorCol;
delta.endLine = cursorLine + 1; // New line will be created
delta.endCol = 0;
// Save the line content before split
delta.lineBeforeSplit = buffer.getLine(cursorLine);
delta.insertedContent = "\n";
delta.deletedContent = "";
delta.lineCountDelta = 1; // One new line
return delta;
}
EditDelta Editor::createDeltaForDeleteSelection()
{
EditDelta delta;
delta.operation = EditDelta::DELETE_TEXT;
// Capture state
delta.preCursorLine = cursorLine;
delta.preCursorCol = cursorCol;
delta.preViewportTop = viewportTop;
delta.preViewportLeft = viewportLeft;
auto [start, end] = getNormalizedSelection();
delta.startLine = start.first;
delta.startCol = start.second;
delta.endLine = end.first;
delta.endCol = end.second;
// Capture the deleted text
delta.deletedContent = getSelectedText();
delta.insertedContent = "";
// Calculate line count change
delta.lineCountDelta = -(end.first - start.first);
return delta;
}
// === Memory Usage Stats ===
size_t Editor::getUndoMemoryUsage() const
{
size_t total = 0;
if (useDeltaUndo_)
{
// Count delta stack
std::stack<DeltaGroup> temp = deltaUndoStack_;
while (!temp.empty())
{
total += temp.top().getMemorySize();
temp.pop();
}
}
else
{
// Count old state stack (approximate)
total = undoStack.size() * sizeof(EditorState);
std::stack<EditorState> temp = undoStack;
while (!temp.empty())
{
total += temp.top().content.capacity();
temp.pop();
}
}
return total;
}
size_t Editor::getRedoMemoryUsage() const
{
size_t total = 0;
if (useDeltaUndo_)
{
std::stack<DeltaGroup> temp = deltaRedoStack_;
while (!temp.empty())
{
total += temp.top().getMemorySize();
temp.pop();
}
}
else
{
total = redoStack.size() * sizeof(EditorState);
std::stack<EditorState> temp = redoStack;
while (!temp.empty())
{
total += temp.top().content.capacity();
temp.pop();
}
}
return total;
}
void Editor::applyDeltaForward(const EditDelta &delta)
{
isUndoRedoing = true;
#ifdef DEBUG_DELTA_UNDO
std::cerr << "Applying delta forward: " << delta.toString() << "\n";
#endif
// Restore cursor to PRE-edit position
cursorLine = delta.preCursorLine;
cursorCol = delta.preCursorCol;
viewportTop = delta.preViewportTop;
viewportLeft = delta.preViewportLeft;
validateCursorAndViewport();
// Notify Tree-sitter BEFORE applying changes
// notifyTreeSitterEdit(delta, false); // false = forward (redo)
switch (delta.operation)
{
case EditDelta::INSERT_CHAR:
case EditDelta::INSERT_TEXT:
{
// Re-insert the text
std::string line = buffer.getLine(cursorLine);
line.insert(cursorCol, delta.insertedContent);
buffer.replaceLine(cursorLine, line);
cursorCol += delta.insertedContent.length();
break;
}
case EditDelta::DELETE_CHAR:
case EditDelta::DELETE_TEXT:
{
// Re-delete the text
if (delta.startLine == delta.endLine)
{
std::string line = buffer.getLine(delta.startLine);
line.erase(delta.startCol, delta.deletedContent.length());
buffer.replaceLine(delta.startLine, line);
}
else
{
// Multi-line deletion
std::string firstLine = buffer.getLine(delta.startLine);
std::string lastLine = buffer.getLine(delta.endLine);
std::string newLine =
firstLine.substr(0, delta.startCol) + lastLine.substr(delta.endCol);
buffer.replaceLine(delta.startLine, newLine);
for (int i = delta.endLine; i > delta.startLine; i--)
{
buffer.deleteLine(i);
}
}
break;
}
case EditDelta::SPLIT_LINE:
{
// Re-split the line
std::string line = buffer.getLine(cursorLine);
std::string leftPart = line.substr(0, cursorCol);
std::string rightPart = line.substr(cursorCol);
buffer.replaceLine(cursorLine, leftPart);
buffer.insertLine(cursorLine + 1, rightPart);
cursorLine++;
cursorCol = 0;
break;
}
case EditDelta::JOIN_LINES:
{
// Re-join the lines
if (delta.startLine + 1 < buffer.getLineCount())
{
std::string firstLine = buffer.getLine(delta.startLine);
std::string secondLine = buffer.getLine(delta.startLine + 1);
buffer.replaceLine(delta.startLine, firstLine + secondLine);
buffer.deleteLine(delta.startLine + 1);
}
break;
}
case EditDelta::REPLACE_LINE:
{
if (!delta.insertedContent.empty())
{
buffer.replaceLine(delta.startLine, delta.insertedContent);
}
break;
}
}
// Restore POST-edit cursor position
cursorLine = delta.postCursorLine;
cursorCol = delta.postCursorCol;
viewportTop = delta.postViewportTop;
viewportLeft = delta.postViewportLeft;
validateCursorAndViewport();
buffer.invalidateLineIndex();
// Invalidate only affected lines (not entire cache)
if (syntaxHighlighter)
{
int startLine = std::min(delta.startLine, delta.preCursorLine);
int endLine = std::max(delta.endLine, delta.postCursorLine);
syntaxHighlighter->invalidateLineRange(startLine,
buffer.getLineCount() - 1);
}
isUndoRedoing = false;
}
// === Apply Delta Reverse (for Undo) ===
void Editor::applyDeltaReverse(const EditDelta &delta)
{
isUndoRedoing = true;
#ifdef DEBUG_DELTA_UNDO
std::cerr << "Applying delta reverse: " << delta.toString() << "\n";
#endif
// Restore cursor to POST-edit position
cursorLine = delta.postCursorLine;
cursorCol = delta.postCursorCol;
viewportTop = delta.postViewportTop;
viewportLeft = delta.postViewportLeft;
validateCursorAndViewport();
switch (delta.operation)
{
case EditDelta::INSERT_CHAR:
case EditDelta::INSERT_TEXT:
{
// Reverse of insert is delete
std::string line = buffer.getLine(delta.startLine);
if (delta.startCol + delta.insertedContent.length() <= line.length())
{
line.erase(delta.startCol, delta.insertedContent.length());
buffer.replaceLine(delta.startLine, line);
}
break;
}
case EditDelta::DELETE_CHAR:
case EditDelta::DELETE_TEXT:
{
// FIXED: Proper multi-line restoration
if (delta.startLine == delta.endLine)
{
// Single line restoration (simple case)
std::string line = buffer.getLine(delta.startLine);
line.insert(delta.startCol, delta.deletedContent);
buffer.replaceLine(delta.startLine, line);
}
else
{
// Multi-line restoration (the bug was here!)
// Step 1: Get the current line at startLine
std::string currentLine = buffer.getLine(delta.startLine);
// Step 2: Split current line at insertion point
std::string beforeInsert = currentLine.substr(0, delta.startCol);
std::string afterInsert = currentLine.substr(delta.startCol);
// Step 3: Split deletedContent by ACTUAL newlines, preserving them
std::vector<std::string> linesToRestore;
size_t pos = 0;
size_t nextNewline;
while ((nextNewline = delta.deletedContent.find('\n', pos)) !=
std::string::npos)
{
// Include everything up to (but not including) the newline
linesToRestore.push_back(
delta.deletedContent.substr(pos, nextNewline - pos));
pos = nextNewline + 1;
}
// Add remaining content (after last newline)
if (pos < delta.deletedContent.length())
{
linesToRestore.push_back(delta.deletedContent.substr(pos));
}
// Step 4: Reconstruct lines correctly
if (!linesToRestore.empty())
{
// First line: beforeInsert + first restored line
buffer.replaceLine(delta.startLine, beforeInsert + linesToRestore[0]);
// Insert middle lines (if any)
for (size_t i = 1; i < linesToRestore.size(); ++i)
{
buffer.insertLine(delta.startLine + i, linesToRestore[i]);
}
// Handle the content after insertion point
if (linesToRestore.size() == 1)
{
// Single line case: append afterInsert to same line
std::string finalLine = buffer.getLine(delta.startLine);
buffer.replaceLine(delta.startLine, finalLine + afterInsert);
}
else
{
// Multi-line case: append afterInsert to last restored line
int lastLineIdx = delta.startLine + linesToRestore.size() - 1;
std::string lastLine = buffer.getLine(lastLineIdx);
buffer.replaceLine(lastLineIdx, lastLine + afterInsert);
}
}
else
{
// Edge case: deletedContent was empty (shouldn't happen, but be safe)
std::cerr << "WARNING: Empty deletedContent in multi-line restore\n";
}
}
break;
}
case EditDelta::SPLIT_LINE:
{
// Reverse of split is join
if (!delta.lineBeforeSplit.empty())
{
if (delta.startLine + 1 < buffer.getLineCount())
{
buffer.replaceLine(delta.startLine, delta.lineBeforeSplit);
buffer.deleteLine(delta.startLine + 1);
}
}
break;
}
case EditDelta::JOIN_LINES:
{
// Reverse of join is split
if (!delta.firstLineBeforeJoin.empty() &&
!delta.secondLineBeforeJoin.empty())
{
buffer.replaceLine(delta.startLine, delta.firstLineBeforeJoin);
buffer.insertLine(delta.startLine + 1, delta.secondLineBeforeJoin);
}
break;
}
case EditDelta::REPLACE_LINE:
{
// Reverse of replace is restore original
if (!delta.deletedContent.empty())
{
buffer.replaceLine(delta.startLine, delta.deletedContent);
}
break;
}
}
// Restore PRE-edit cursor position
cursorLine = delta.preCursorLine;
cursorCol = delta.preCursorCol;
viewportTop = delta.preViewportTop;
viewportLeft = delta.preViewportLeft;
validateCursorAndViewport();
buffer.invalidateLineIndex();
isUndoRedoing = false;
}
// Fallback
void Editor::saveState()
{
if (isSaving || isUndoRedoing)
return;
auto now = std::chrono::steady_clock::now();
auto elapsed =
std::chrono::duration_cast<std::chrono::milliseconds>(now - lastEditTime)
.count();
// Only save if enough time has passed since last edit
if (elapsed > UNDO_GROUP_TIMEOUT_MS || undoStack.empty())
{
EditorState state = getCurrentState();
undoStack.push(state);
limitUndoStack();
while (!redoStack.empty())
{
redoStack.pop();
}
}
lastEditTime = now;
}
void Editor::optimizedLineInvalidation(int startLine, int endLine)
{
if (!syntaxHighlighter)
{
return;
}
// Only invalidate if change is significant
int changeSize = endLine - startLine + 1;
if (changeSize > 100)
{
// Large change: full reparse (but async)
syntaxHighlighter->clearAllCache();
syntaxHighlighter->scheduleBackgroundParse(buffer);
}
else if (changeSize > 10)
{
// Medium change: invalidate range and reparse viewport
syntaxHighlighter->invalidateLineRange(startLine,
buffer.getLineCount() - 1);
syntaxHighlighter->parseViewportOnly(buffer, viewportTop);
}
else
{
// Small change: just invalidate the affected lines
syntaxHighlighter->invalidateLineRange(startLine, endLine);
}
}
// ============================================================================
// FIX 4: Add Tree-sitter Edit Notification for Delta Operations
// ============================================================================
// Add this method to track Tree-sitter edits during delta apply:
void Editor::notifyTreeSitterEdit(const EditDelta &delta, bool isReverse)
{
if (!syntaxHighlighter)
{
return;
}
// Calculate byte positions
size_t start_byte = buffer.lineColToPos(delta.startLine, delta.startCol);
if (isReverse)
{
// Undoing: reverse the original operation
switch (delta.operation)
{
case EditDelta::INSERT_CHAR:
case EditDelta::INSERT_TEXT:
{
// Was an insert, now delete
size_t len = delta.insertedContent.length();
syntaxHighlighter->notifyEdit(start_byte, 0, len, // Inserting back
delta.startLine, delta.startCol,
delta.startLine, delta.startCol,
delta.postCursorLine, delta.postCursorCol);
break;
}
case EditDelta::DELETE_CHAR:
case EditDelta::DELETE_TEXT:
{
// Was a delete, now insert
size_t len = delta.deletedContent.length();
syntaxHighlighter->notifyEdit(start_byte, len, 0, // Deleting
delta.startLine, delta.startCol,
delta.endLine, delta.endCol,
delta.startLine, delta.startCol);
break;
}
case EditDelta::SPLIT_LINE:
{
// Was a split, now join
syntaxHighlighter->notifyEdit(start_byte, 0, 1, // Remove newline
delta.startLine, delta.startCol,
delta.startLine, delta.startCol,
delta.startLine + 1, 0);
break;
}
case EditDelta::JOIN_LINES:
{
// Was a join, now split
syntaxHighlighter->notifyEdit(start_byte, 1, 0, // Add newline
delta.startLine, delta.startCol,
delta.startLine + 1, 0, delta.startLine,
delta.startCol);
break;
}
}
}
else
{
// Redoing: apply the original operation
switch (delta.operation)
{
case EditDelta::INSERT_CHAR:
case EditDelta::INSERT_TEXT:
{
size_t len = delta.insertedContent.length();
syntaxHighlighter->notifyEdit(start_byte, len, 0, delta.startLine,
delta.startCol, delta.postCursorLine,
delta.postCursorCol, delta.startLine,
delta.startCol);
break;
}
case EditDelta::DELETE_CHAR:
case EditDelta::DELETE_TEXT:
{
size_t len = delta.deletedContent.length();
syntaxHighlighter->notifyEdit(
start_byte, 0, len, delta.startLine, delta.startCol, delta.startLine,
delta.startCol, delta.endLine, delta.endCol);
break;
}
case EditDelta::SPLIT_LINE:
{
syntaxHighlighter->notifyEdit(start_byte, 1, 0, delta.startLine,
delta.startCol, delta.startLine + 1, 0,
delta.startLine, delta.startCol);
break;
}
case EditDelta::JOIN_LINES:
{
syntaxHighlighter->notifyEdit(start_byte, 0, 1, delta.startLine,
delta.startCol, delta.startLine,
delta.startCol, delta.startLine + 1, 0);
break;
}
}
}
}
```