mirror of
https://github.com/neovim/neovim
synced 2025-07-15 08:41:47 +00:00
feat(lsp): support multiline semantic tokens #34458
This commit is contained in:
Riley Bruins
GitHub
@ -2187,6 +2187,7 @@ get_at_pos({bufnr}, {row}, {col})
|
||||
fields:
|
||||
• line (integer) line number, 0-based
|
||||
• start_col (integer) start column, 0-based
|
||||
• end_line (integer) end line number, 0-based
|
||||
• end_col (integer) end column, 0-based
|
||||
• type (string) token type as string, e.g. "variable"
|
||||
• modifiers (table) token modifiers as a set. E.g., { static = true,
|
||||
|
||||
@ -173,6 +173,7 @@ LSP
|
||||
https://microsoft.github.io/language-server-protocol/specifications/specification-current/#workspace_dagnostics
|
||||
• Incremental selection is now supported via `textDocument/selectionRange`.
|
||||
`an` selects outwards and `in` selects inwards.
|
||||
• Support for multiline semantic tokens.
|
||||
|
||||
LUA
|
||||
|
||||
|
||||
@ -404,8 +404,7 @@ function protocol.make_client_capabilities()
|
||||
},
|
||||
|
||||
overlappingTokenSupport = true,
|
||||
-- TODO(jdrouhard): Add support for this
|
||||
multilineTokenSupport = false,
|
||||
multilineTokenSupport = true,
|
||||
serverCancelSupport = false,
|
||||
augmentsSyntaxTokens = true,
|
||||
},
|
||||
|
||||
@ -2,11 +2,13 @@ local api = vim.api
|
||||
local bit = require('bit')
|
||||
local ms = require('vim.lsp.protocol').Methods
|
||||
local util = require('vim.lsp.util')
|
||||
local Range = require('vim.treesitter._range')
|
||||
local uv = vim.uv
|
||||
|
||||
--- @class (private) STTokenRange
|
||||
--- @field line integer line number 0-based
|
||||
--- @field start_col integer start column 0-based
|
||||
--- @field end_line integer end line number 0-based
|
||||
--- @field end_col integer end column 0-based
|
||||
--- @field type string token type as string
|
||||
--- @field modifiers table<string,boolean> token modifiers as a set. E.g., { static = true, readonly = true }
|
||||
@ -44,7 +46,7 @@ local STHighlighter = { active = {} }
|
||||
local function lower_bound(tokens, line, lo, hi)
|
||||
while lo < hi do
|
||||
local mid = bit.rshift(lo + hi, 1) -- Equivalent to floor((lo + hi) / 2).
|
||||
if tokens[mid].line < line then
|
||||
if tokens[mid].end_line < line then
|
||||
lo = mid + 1
|
||||
else
|
||||
hi = mid
|
||||
@ -102,6 +104,8 @@ local function tokens_to_ranges(data, bufnr, client, request)
|
||||
local token_modifiers = legend.tokenModifiers
|
||||
local encoding = client.offset_encoding
|
||||
local lines = api.nvim_buf_get_lines(bufnr, 0, -1, false)
|
||||
-- For all encodings, \r\n takes up two code points, and \n (or \r) takes up one.
|
||||
local eol_offset = vim.bo.fileformat[bufnr] == 'dos' and 2 or 1
|
||||
local ranges = {} ---@type STTokenRange[]
|
||||
|
||||
local start = uv.hrtime()
|
||||
@ -141,11 +145,23 @@ local function tokens_to_ranges(data, bufnr, client, request)
|
||||
if token_type then
|
||||
local modifiers = modifiers_from_number(data[i + 4], token_modifiers)
|
||||
local end_char = start_char + data[i + 2] --- @type integer LuaLS bug
|
||||
local buf_line = lines and lines[line + 1] or ''
|
||||
local buf_line = lines[line + 1] or ''
|
||||
local end_line = line ---@type integer
|
||||
local start_col = vim.str_byteindex(buf_line, encoding, start_char, false)
|
||||
local end_col = vim.str_byteindex(buf_line, encoding, end_char, false)
|
||||
|
||||
end_char = end_char - vim.str_utfindex(buf_line, encoding) - eol_offset
|
||||
-- While end_char goes past the given line, extend the token range to the next line
|
||||
while end_char > 0 do
|
||||
end_line = end_line + 1
|
||||
buf_line = lines[end_line + 1] or ''
|
||||
end_col = vim.str_byteindex(buf_line, encoding, end_char, false)
|
||||
end_char = end_char - vim.str_utfindex(buf_line, encoding) - eol_offset
|
||||
end
|
||||
|
||||
ranges[#ranges + 1] = {
|
||||
line = line,
|
||||
end_line = end_line,
|
||||
start_col = start_col,
|
||||
end_col = end_col,
|
||||
type = token_type,
|
||||
@ -398,6 +414,7 @@ end
|
||||
local function set_mark(bufnr, ns, token, hl_group, priority)
|
||||
vim.api.nvim_buf_set_extmark(bufnr, ns, token.line, token.start_col, {
|
||||
hl_group = hl_group,
|
||||
end_line = token.end_line,
|
||||
end_col = token.end_col,
|
||||
priority = priority,
|
||||
strict = false,
|
||||
@ -692,6 +709,7 @@ end
|
||||
--- the following fields:
|
||||
--- - line (integer) line number, 0-based
|
||||
--- - start_col (integer) start column, 0-based
|
||||
--- - end_line (integer) end line number, 0-based
|
||||
--- - end_col (integer) end column, 0-based
|
||||
--- - type (string) token type as string, e.g. "variable"
|
||||
--- - modifiers (table) token modifiers as a set. E.g., { static = true, readonly = true }
|
||||
@ -709,6 +727,8 @@ function M.get_at_pos(bufnr, row, col)
|
||||
row, col = cursor[1] - 1, cursor[2]
|
||||
end
|
||||
|
||||
local position = { row, col, row, col }
|
||||
|
||||
local tokens = {} --- @type STTokenRangeInspect[]
|
||||
for client_id, client in pairs(highlighter.client_state) do
|
||||
local highlights = client.current_result.highlights
|
||||
@ -722,7 +742,9 @@ function M.get_at_pos(bufnr, row, col)
|
||||
break
|
||||
end
|
||||
|
||||
if token.start_col <= col and token.end_col > col then
|
||||
if
|
||||
Range.contains({ token.line, token.start_col, token.end_line, token.end_col }, position)
|
||||
then
|
||||
token.client_id = client_id
|
||||
tokens[#tokens + 1] = token
|
||||
end
|
||||
|
||||
@ -132,6 +132,53 @@ describe('semantic token highlighting', function()
|
||||
}
|
||||
end)
|
||||
|
||||
it('buffer is highlighted with multiline tokens', function()
|
||||
insert(text)
|
||||
exec_lua(function()
|
||||
_G.server2 = _G._create_server({
|
||||
capabilities = {
|
||||
semanticTokensProvider = {
|
||||
full = { delta = true },
|
||||
legend = vim.fn.json_decode(legend),
|
||||
},
|
||||
},
|
||||
handlers = {
|
||||
['textDocument/semanticTokens/full'] = function(_, _, callback)
|
||||
callback(nil, {
|
||||
data = { 5, 0, 82, 0, 0 },
|
||||
resultId = 1,
|
||||
})
|
||||
end,
|
||||
},
|
||||
})
|
||||
end, legend, response, edit_response)
|
||||
exec_lua(function()
|
||||
local bufnr = vim.api.nvim_get_current_buf()
|
||||
vim.api.nvim_win_set_buf(0, bufnr)
|
||||
vim.bo[bufnr].filetype = 'some-filetype'
|
||||
vim.lsp.start({ name = 'dummy', cmd = _G.server2.cmd })
|
||||
end)
|
||||
|
||||
screen:expect {
|
||||
grid = [[
|
||||
#include <iostream> |
|
||||
|
|
||||
int main() |
|
||||
{ |
|
||||
int x; |
|
||||
{2:#ifdef __cplusplus} |
|
||||
{2: std::cout << x << "\n";} |
|
||||
{2:#else} |
|
||||
{2: printf("%d\n", x);} |
|
||||
{2:#endif} |
|
||||
} |
|
||||
^} |
|
||||
{1:~ }|*3
|
||||
|
|
||||
]],
|
||||
}
|
||||
end)
|
||||
|
||||
it('use LspTokenUpdate and highlight_token', function()
|
||||
insert(text)
|
||||
exec_lua(function()
|
||||
@ -727,6 +774,7 @@ describe('semantic token highlighting', function()
|
||||
expected = {
|
||||
{
|
||||
line = 0,
|
||||
end_line = 0,
|
||||
modifiers = { declaration = true, globalScope = true },
|
||||
start_col = 6,
|
||||
end_col = 9,
|
||||
@ -768,6 +816,7 @@ int main()
|
||||
expected = {
|
||||
{ -- main
|
||||
line = 1,
|
||||
end_line = 1,
|
||||
modifiers = { declaration = true, globalScope = true },
|
||||
start_col = 4,
|
||||
end_col = 8,
|
||||
@ -776,6 +825,7 @@ int main()
|
||||
},
|
||||
{ -- __cplusplus
|
||||
line = 3,
|
||||
end_line = 3,
|
||||
modifiers = { globalScope = true },
|
||||
start_col = 9,
|
||||
end_col = 20,
|
||||
@ -784,6 +834,7 @@ int main()
|
||||
},
|
||||
{ -- x
|
||||
line = 4,
|
||||
end_line = 4,
|
||||
modifiers = { declaration = true, readonly = true, functionScope = true },
|
||||
start_col = 12,
|
||||
end_col = 13,
|
||||
@ -792,6 +843,7 @@ int main()
|
||||
},
|
||||
{ -- std
|
||||
line = 5,
|
||||
end_line = 5,
|
||||
modifiers = { defaultLibrary = true, globalScope = true },
|
||||
start_col = 2,
|
||||
end_col = 5,
|
||||
@ -800,6 +852,7 @@ int main()
|
||||
},
|
||||
{ -- cout
|
||||
line = 5,
|
||||
end_line = 5,
|
||||
modifiers = { defaultLibrary = true, globalScope = true },
|
||||
start_col = 7,
|
||||
end_col = 11,
|
||||
@ -808,6 +861,7 @@ int main()
|
||||
},
|
||||
{ -- x
|
||||
line = 5,
|
||||
end_line = 5,
|
||||
modifiers = { readonly = true, functionScope = true },
|
||||
start_col = 15,
|
||||
end_col = 16,
|
||||
@ -816,6 +870,7 @@ int main()
|
||||
},
|
||||
{ -- std
|
||||
line = 5,
|
||||
end_line = 5,
|
||||
modifiers = { defaultLibrary = true, globalScope = true },
|
||||
start_col = 20,
|
||||
end_col = 23,
|
||||
@ -824,6 +879,7 @@ int main()
|
||||
},
|
||||
{ -- endl
|
||||
line = 5,
|
||||
end_line = 5,
|
||||
modifiers = { defaultLibrary = true, globalScope = true },
|
||||
start_col = 25,
|
||||
end_col = 29,
|
||||
@ -832,6 +888,7 @@ int main()
|
||||
},
|
||||
{ -- #else comment #endif
|
||||
line = 6,
|
||||
end_line = 6,
|
||||
modifiers = {},
|
||||
start_col = 0,
|
||||
end_col = 7,
|
||||
@ -840,6 +897,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 7,
|
||||
end_line = 7,
|
||||
modifiers = {},
|
||||
start_col = 0,
|
||||
end_col = 11,
|
||||
@ -848,6 +906,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 8,
|
||||
end_line = 8,
|
||||
modifiers = {},
|
||||
start_col = 0,
|
||||
end_col = 8,
|
||||
@ -891,6 +950,7 @@ b = "as"]],
|
||||
expected = {
|
||||
{
|
||||
line = 0,
|
||||
end_line = 0,
|
||||
modifiers = {},
|
||||
start_col = 0,
|
||||
end_col = 10,
|
||||
@ -899,6 +959,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 1,
|
||||
end_line = 1,
|
||||
modifiers = { declaration = true }, -- a
|
||||
start_col = 6,
|
||||
end_col = 7,
|
||||
@ -907,6 +968,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 2,
|
||||
end_line = 2,
|
||||
modifiers = { static = true }, -- b (global)
|
||||
start_col = 0,
|
||||
end_col = 1,
|
||||
@ -950,6 +1012,7 @@ b = "as"]],
|
||||
expected = {
|
||||
{
|
||||
line = 0,
|
||||
end_line = 0,
|
||||
modifiers = {},
|
||||
start_col = 0,
|
||||
end_col = 3, -- pub
|
||||
@ -958,6 +1021,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 0,
|
||||
end_line = 0,
|
||||
modifiers = {},
|
||||
start_col = 4,
|
||||
end_col = 6, -- fn
|
||||
@ -966,6 +1030,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 0,
|
||||
end_line = 0,
|
||||
modifiers = { declaration = true, public = true },
|
||||
start_col = 7,
|
||||
end_col = 11, -- main
|
||||
@ -974,6 +1039,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 0,
|
||||
end_line = 0,
|
||||
modifiers = {},
|
||||
start_col = 11,
|
||||
end_col = 12,
|
||||
@ -982,6 +1048,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 0,
|
||||
end_line = 0,
|
||||
modifiers = {},
|
||||
start_col = 12,
|
||||
end_col = 13,
|
||||
@ -990,6 +1057,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 0,
|
||||
end_line = 0,
|
||||
modifiers = {},
|
||||
start_col = 14,
|
||||
end_col = 15,
|
||||
@ -998,6 +1066,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 1,
|
||||
end_line = 1,
|
||||
modifiers = {},
|
||||
start_col = 4,
|
||||
end_col = 12,
|
||||
@ -1006,6 +1075,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 1,
|
||||
end_line = 1,
|
||||
modifiers = {},
|
||||
start_col = 12,
|
||||
end_col = 13,
|
||||
@ -1014,6 +1084,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 1,
|
||||
end_line = 1,
|
||||
modifiers = {},
|
||||
start_col = 13,
|
||||
end_col = 27,
|
||||
@ -1022,6 +1093,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 1,
|
||||
end_line = 1,
|
||||
modifiers = {},
|
||||
start_col = 27,
|
||||
end_col = 28,
|
||||
@ -1030,6 +1102,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 1,
|
||||
end_line = 1,
|
||||
modifiers = {},
|
||||
start_col = 28,
|
||||
end_col = 29,
|
||||
@ -1038,6 +1111,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 2,
|
||||
end_line = 2,
|
||||
modifiers = { controlFlow = true },
|
||||
start_col = 4,
|
||||
end_col = 9, -- break
|
||||
@ -1046,6 +1120,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 2,
|
||||
end_line = 2,
|
||||
modifiers = {},
|
||||
start_col = 10,
|
||||
end_col = 14, -- rust
|
||||
@ -1054,6 +1129,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 2,
|
||||
end_line = 2,
|
||||
modifiers = {},
|
||||
start_col = 14,
|
||||
end_col = 15,
|
||||
@ -1062,6 +1138,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 3,
|
||||
end_line = 3,
|
||||
modifiers = { documentation = true },
|
||||
start_col = 4,
|
||||
end_col = 13,
|
||||
@ -1070,6 +1147,7 @@ b = "as"]],
|
||||
},
|
||||
{
|
||||
line = 4,
|
||||
end_line = 4,
|
||||
modifiers = {},
|
||||
start_col = 0,
|
||||
end_col = 1,
|
||||
@ -1151,6 +1229,7 @@ b = "as"]],
|
||||
globalScope = true,
|
||||
},
|
||||
start_col = 6,
|
||||
end_line = 0,
|
||||
end_col = 9,
|
||||
type = 'variable',
|
||||
marked = true,
|
||||
@ -1164,6 +1243,7 @@ b = "as"]],
|
||||
globalScope = true,
|
||||
},
|
||||
start_col = 6,
|
||||
end_line = 1,
|
||||
end_col = 9,
|
||||
type = 'variable',
|
||||
marked = true,
|
||||
@ -1234,6 +1314,7 @@ int main()
|
||||
expected1 = {
|
||||
{
|
||||
line = 2,
|
||||
end_line = 2,
|
||||
start_col = 4,
|
||||
end_col = 8,
|
||||
modifiers = { declaration = true, globalScope = true },
|
||||
@ -1242,6 +1323,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 4,
|
||||
end_line = 4,
|
||||
start_col = 8,
|
||||
end_col = 9,
|
||||
modifiers = { declaration = true, functionScope = true },
|
||||
@ -1250,6 +1332,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 5,
|
||||
end_line = 5,
|
||||
start_col = 7,
|
||||
end_col = 18,
|
||||
modifiers = { globalScope = true },
|
||||
@ -1258,6 +1341,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 6,
|
||||
end_line = 6,
|
||||
start_col = 4,
|
||||
end_col = 7,
|
||||
modifiers = { defaultLibrary = true, globalScope = true },
|
||||
@ -1266,6 +1350,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 6,
|
||||
end_line = 6,
|
||||
start_col = 9,
|
||||
end_col = 13,
|
||||
modifiers = { defaultLibrary = true, globalScope = true },
|
||||
@ -1274,6 +1359,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 6,
|
||||
end_line = 6,
|
||||
start_col = 17,
|
||||
end_col = 18,
|
||||
marked = true,
|
||||
@ -1282,6 +1368,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 7,
|
||||
end_line = 7,
|
||||
start_col = 0,
|
||||
end_col = 5,
|
||||
marked = true,
|
||||
@ -1290,6 +1377,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 8,
|
||||
end_line = 8,
|
||||
end_col = 22,
|
||||
modifiers = {},
|
||||
start_col = 0,
|
||||
@ -1298,6 +1386,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 9,
|
||||
end_line = 9,
|
||||
start_col = 0,
|
||||
end_col = 6,
|
||||
modifiers = {},
|
||||
@ -1308,6 +1397,7 @@ int main()
|
||||
expected2 = {
|
||||
{
|
||||
line = 2,
|
||||
end_line = 2,
|
||||
start_col = 4,
|
||||
end_col = 8,
|
||||
modifiers = { declaration = true, globalScope = true },
|
||||
@ -1316,6 +1406,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 4,
|
||||
end_line = 4,
|
||||
start_col = 8,
|
||||
end_col = 9,
|
||||
modifiers = { declaration = true, globalScope = true },
|
||||
@ -1324,6 +1415,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 5,
|
||||
end_line = 5,
|
||||
end_col = 12,
|
||||
start_col = 11,
|
||||
modifiers = { declaration = true, functionScope = true },
|
||||
@ -1332,6 +1424,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 6,
|
||||
end_line = 6,
|
||||
start_col = 7,
|
||||
end_col = 18,
|
||||
modifiers = { globalScope = true },
|
||||
@ -1340,6 +1433,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 7,
|
||||
end_line = 7,
|
||||
start_col = 4,
|
||||
end_col = 7,
|
||||
modifiers = { defaultLibrary = true, globalScope = true },
|
||||
@ -1348,6 +1442,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 7,
|
||||
end_line = 7,
|
||||
start_col = 9,
|
||||
end_col = 13,
|
||||
modifiers = { defaultLibrary = true, globalScope = true },
|
||||
@ -1356,6 +1451,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 7,
|
||||
end_line = 7,
|
||||
start_col = 17,
|
||||
end_col = 18,
|
||||
marked = true,
|
||||
@ -1364,6 +1460,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 8,
|
||||
end_line = 8,
|
||||
start_col = 0,
|
||||
end_col = 5,
|
||||
marked = true,
|
||||
@ -1372,6 +1469,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 9,
|
||||
end_line = 9,
|
||||
end_col = 22,
|
||||
modifiers = {},
|
||||
start_col = 0,
|
||||
@ -1380,6 +1478,7 @@ int main()
|
||||
},
|
||||
{
|
||||
line = 10,
|
||||
end_line = 10,
|
||||
start_col = 0,
|
||||
end_col = 6,
|
||||
modifiers = {},
|
||||
@ -1444,6 +1543,7 @@ int main()
|
||||
expected1 = {
|
||||
{
|
||||
line = 0,
|
||||
end_line = 0,
|
||||
modifiers = {
|
||||
declaration = true,
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user