GitHub/neovim
1
0
Fork 0
mirror of https://github.com/neovim/neovim synced 2025-07-15 16:51:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(terminal): support grapheme clusters, including emoji

Browse Source
This commit is contained in:
bfredl
2024-12-18 14:49:38 +01:00
parent 9d9ee3476e
commit e3bfcf2fd4
11 changed files with 184 additions and 631 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
runtime/doc/news.txt
View File

@ -370,6 +370,9 @@ These existing features changed their behavior.
more emoji characters than before, including those encoded with multiple
emoji codepoints combined with ZWJ (zero width joiner) codepoints.
This also applies to :terminal output, where width of cells will be calculated
using the upgraded implementation.
• Custom highlights in 'rulerformat', 'statuscolumn', 'statusline', 'tabline',
'winbar' and the number column (through |:sign-define| `numhl`) now combine
with their respective highlight groups, as opposed to |hl-Normal|.

13
src/nvim/terminal.c
View File

@ -65,6 +65,7 @@
#include "nvim/ex_docmd.h"
#include "nvim/getchar.h"
#include "nvim/globals.h"
#include "nvim/grid.h"
#include "nvim/highlight.h"
#include "nvim/highlight_defs.h"
#include "nvim/highlight_group.h"
@ -1347,7 +1348,7 @@ static int term_sb_pop(int cols, VTermScreenCell *cells, void *data)
// copy to vterm state
memcpy(cells, sbrow->cells, sizeof(cells[0]) * cols_to_copy);
for (size_t col = cols_to_copy; col < (size_t)cols; col++) {
cells[col].chars[0] = 0;
cells[col].schar = 0;
cells[col].width = 1;
}
@ -1857,12 +1858,8 @@ static void fetch_row(Terminal *term, int row, int end_col)
while (col < end_col) {
VTermScreenCell cell;
fetch_cell(term, row, col, &cell);
if (cell.chars[0]) {
int cell_len = 0;
for (int i = 0; i < VTERM_MAX_CHARS_PER_CELL && cell.chars[i]; i++) {
cell_len += utf_char2bytes((int)cell.chars[i], ptr + cell_len);
}
ptr += cell_len;
if (cell.schar) {
schar_get_adv(&ptr, cell.schar);
line_len = (size_t)(ptr - term->textbuf);
} else {
*ptr++ = ' ';
@ -1883,7 +1880,7 @@ static bool fetch_cell(Terminal *term, int row, int col, VTermScreenCell *cell)
} else {
// fill the pointer with an empty cell
*cell = (VTermScreenCell) {
.chars = { 0 },
.schar = 0,
.width = 1,
};
return false;

111
src/vterm/fullwidth.inc
View File

@ -1,111 +0,0 @@
{ 0x1100, 0x115f },
{ 0x231a, 0x231b },
{ 0x2329, 0x232a },
{ 0x23e9, 0x23ec },
{ 0x23f0, 0x23f0 },
{ 0x23f3, 0x23f3 },
{ 0x25fd, 0x25fe },
{ 0x2614, 0x2615 },
{ 0x2648, 0x2653 },
{ 0x267f, 0x267f },
{ 0x2693, 0x2693 },
{ 0x26a1, 0x26a1 },
{ 0x26aa, 0x26ab },
{ 0x26bd, 0x26be },
{ 0x26c4, 0x26c5 },
{ 0x26ce, 0x26ce },
{ 0x26d4, 0x26d4 },
{ 0x26ea, 0x26ea },
{ 0x26f2, 0x26f3 },
{ 0x26f5, 0x26f5 },
{ 0x26fa, 0x26fa },
{ 0x26fd, 0x26fd },
{ 0x2705, 0x2705 },
{ 0x270a, 0x270b },
{ 0x2728, 0x2728 },
{ 0x274c, 0x274c },
{ 0x274e, 0x274e },
{ 0x2753, 0x2755 },
{ 0x2757, 0x2757 },
{ 0x2795, 0x2797 },
{ 0x27b0, 0x27b0 },
{ 0x27bf, 0x27bf },
{ 0x2b1b, 0x2b1c },
{ 0x2b50, 0x2b50 },
{ 0x2b55, 0x2b55 },
{ 0x2e80, 0x2e99 },
{ 0x2e9b, 0x2ef3 },
{ 0x2f00, 0x2fd5 },
{ 0x2ff0, 0x2ffb },
{ 0x3000, 0x303e },
{ 0x3041, 0x3096 },
{ 0x3099, 0x30ff },
{ 0x3105, 0x312f },
{ 0x3131, 0x318e },
{ 0x3190, 0x31ba },
{ 0x31c0, 0x31e3 },
{ 0x31f0, 0x321e },
{ 0x3220, 0x3247 },
{ 0x3250, 0x4dbf },
{ 0x4e00, 0xa48c },
{ 0xa490, 0xa4c6 },
{ 0xa960, 0xa97c },
{ 0xac00, 0xd7a3 },
{ 0xf900, 0xfaff },
{ 0xfe10, 0xfe19 },
{ 0xfe30, 0xfe52 },
{ 0xfe54, 0xfe66 },
{ 0xfe68, 0xfe6b },
{ 0xff01, 0xff60 },
{ 0xffe0, 0xffe6 },
{ 0x16fe0, 0x16fe3 },
{ 0x17000, 0x187f7 },
{ 0x18800, 0x18af2 },
{ 0x1b000, 0x1b11e },
{ 0x1b150, 0x1b152 },
{ 0x1b164, 0x1b167 },
{ 0x1b170, 0x1b2fb },
{ 0x1f004, 0x1f004 },
{ 0x1f0cf, 0x1f0cf },
{ 0x1f18e, 0x1f18e },
{ 0x1f191, 0x1f19a },
{ 0x1f200, 0x1f202 },
{ 0x1f210, 0x1f23b },
{ 0x1f240, 0x1f248 },
{ 0x1f250, 0x1f251 },
{ 0x1f260, 0x1f265 },
{ 0x1f300, 0x1f320 },
{ 0x1f32d, 0x1f335 },
{ 0x1f337, 0x1f37c },
{ 0x1f37e, 0x1f393 },
{ 0x1f3a0, 0x1f3ca },
{ 0x1f3cf, 0x1f3d3 },
{ 0x1f3e0, 0x1f3f0 },
{ 0x1f3f4, 0x1f3f4 },
{ 0x1f3f8, 0x1f43e },
{ 0x1f440, 0x1f440 },
{ 0x1f442, 0x1f4fc },
{ 0x1f4ff, 0x1f53d },
{ 0x1f54b, 0x1f54e },
{ 0x1f550, 0x1f567 },
{ 0x1f57a, 0x1f57a },
{ 0x1f595, 0x1f596 },
{ 0x1f5a4, 0x1f5a4 },
{ 0x1f5fb, 0x1f64f },
{ 0x1f680, 0x1f6c5 },
{ 0x1f6cc, 0x1f6cc },
{ 0x1f6d0, 0x1f6d2 },
{ 0x1f6d5, 0x1f6d5 },
{ 0x1f6eb, 0x1f6ec },
{ 0x1f6f4, 0x1f6fa },
{ 0x1f7e0, 0x1f7eb },
{ 0x1f90d, 0x1f971 },
{ 0x1f973, 0x1f976 },
{ 0x1f97a, 0x1f9a2 },
{ 0x1f9a5, 0x1f9aa },
{ 0x1f9ae, 0x1f9ca },
{ 0x1f9cd, 0x1f9ff },
{ 0x1fa70, 0x1fa73 },
{ 0x1fa78, 0x1fa7a },
{ 0x1fa80, 0x1fa82 },
{ 0x1fa90, 0x1fa95 },

78
src/vterm/screen.c
View File

@ -2,6 +2,7 @@
#include <stdio.h>
#include <string.h>
#include "nvim/grid.h"
#include "nvim/mbyte.h"
#include "nvim/tui/termkey/termkey.h"
@ -41,7 +42,7 @@ typedef struct
/* Internal representation of a screen cell */
typedef struct
{
uint32_t chars[VTERM_MAX_CHARS_PER_CELL];
schar_T schar;
ScreenPen pen;
} ScreenCell;
@ -79,7 +80,7 @@ struct VTermScreen
static inline void clearcell(const VTermScreen *screen, ScreenCell *cell)
{
cell->chars[0] = 0;
cell->schar = 0;
cell->pen = screen->pen;
}
@ -182,16 +183,13 @@ static int putglyph(VTermGlyphInfo *info, VTermPos pos, void *user)
if(!cell)
return 0;
int i;
for(i = 0; i < VTERM_MAX_CHARS_PER_CELL && info->chars[i]; i++) {
cell->chars[i] = info->chars[i];
cell->schar = info->schar;
if (info->schar != 0) {
cell->pen = screen->pen;
}
if(i < VTERM_MAX_CHARS_PER_CELL)
cell->chars[i] = 0;
for(int col = 1; col < info->width; col++)
getcell(screen, pos.row, pos.col + col)->chars[0] = (uint32_t)-1;
getcell(screen, pos.row, pos.col + col)->schar = (uint32_t)-1;
VTermRect rect = {
.start_row = pos.row,
@ -284,7 +282,7 @@ static int erase_internal(VTermRect rect, int selective, void *user)
if(selective && cell->pen.protected_cell)
continue;
cell->chars[0] = 0;
cell->schar = 0;
cell->pen = (ScreenPen){
/* Only copy .fg and .bg; leave things like rv in reset state */
.fg = screen->pen.fg,
@ -504,7 +502,7 @@ static int bell(void *user)
static int line_popcount(ScreenCell *buffer, int row, int rows, int cols)
{
int col = cols - 1;
while(col >= 0 && buffer[row * cols + col].chars[0] == 0)
while(col >= 0 && buffer[row * cols + col].schar == 0)
col--;
return col + 1;
}
@ -690,11 +688,7 @@ static void resize_buffer(VTermScreen *screen, int bufidx, int new_rows, int new
VTermScreenCell *src = &screen->sb_buffer[pos.col];
ScreenCell *dst = &new_buffer[pos.row * new_cols + pos.col];
for(int i = 0; i < VTERM_MAX_CHARS_PER_CELL; i++) {
dst->chars[i] = src->chars[i];
if(!src->chars[i])
break;
}
dst->schar = src->schar;
dst->pen.bold = src->attrs.bold;
dst->pen.underline = src->attrs.underline;
@ -713,7 +707,7 @@ static void resize_buffer(VTermScreen *screen, int bufidx, int new_rows, int new
dst->pen.uri = src->uri;
if(src->width == 2 && pos.col < (new_cols-1))
(dst + 1)->chars[0] = (uint32_t) -1;
(dst + 1)->schar = (uint32_t) -1;
}
for( ; pos.col < new_cols; pos.col++)
clearcell(screen, &new_buffer[pos.row * new_cols + pos.col]);
@ -914,49 +908,41 @@ void vterm_screen_reset(VTermScreen *screen, int hard)
vterm_screen_flush_damage(screen);
}
static size_t _get_chars(const VTermScreen *screen, const int utf8, void *buffer, size_t len, const VTermRect rect)
size_t vterm_screen_get_text(const VTermScreen *screen, char *buffer, size_t len, const VTermRect rect)
{
size_t outpos = 0;
int padding = 0;
#define PUT(c) \
if(utf8) { \
size_t thislen = utf_char2len(c); \
#define PUT(bytes, thislen) \
if(true) { \
if(buffer && outpos + thislen <= len) \
outpos += fill_utf8((c), (char *)buffer + outpos); \
else \
outpos += thislen; \
memcpy((char *)buffer + outpos, bytes, thislen); \
outpos += thislen; \
} \
else { \
if(buffer && outpos + 1 <= len) \
((uint32_t*)buffer)[outpos++] = (c); \
else \
outpos++; \
}
for(int row = rect.start_row; row < rect.end_row; row++) {
for(int col = rect.start_col; col < rect.end_col; col++) {
ScreenCell *cell = getcell(screen, row, col);
if(cell->chars[0] == 0)
if(cell->schar == 0)
// Erased cell, might need a space
padding++;
else if(cell->chars[0] == (uint32_t)-1)
else if(cell->schar == (uint32_t)-1)
// Gap behind a double-width char, do nothing
;
else {
while(padding) {
PUT(UNICODE_SPACE);
PUT(" ", 1);
padding--;
}
for(int i = 0; i < VTERM_MAX_CHARS_PER_CELL && cell->chars[i]; i++) {
PUT(cell->chars[i]);
}
char buf[MAX_SCHAR_SIZE + 1];
size_t thislen = schar_get(buf, cell->schar);
PUT(buf, thislen);
}
}
if(row < rect.end_row - 1) {
PUT(UNICODE_LINEFEED);
PUT("\n", 1);
padding = 0;
}
}
@ -964,16 +950,6 @@ static size_t _get_chars(const VTermScreen *screen, const int utf8, void *buffer
return outpos;
}
size_t vterm_screen_get_chars(const VTermScreen *screen, uint32_t *chars, size_t len, const VTermRect rect)
{
return _get_chars(screen, 0, chars, len, rect);
}
size_t vterm_screen_get_text(const VTermScreen *screen, char *str, size_t len, const VTermRect rect)
{
return _get_chars(screen, 1, str, len, rect);
}
/* Copy internal to external representation of a screen cell */
int vterm_screen_get_cell(const VTermScreen *screen, VTermPos pos, VTermScreenCell *cell)
{
@ -981,11 +957,7 @@ int vterm_screen_get_cell(const VTermScreen *screen, VTermPos pos, VTermScreenCe
if(!intcell)
return 0;
for(int i = 0; i < VTERM_MAX_CHARS_PER_CELL; i++) {
cell->chars[i] = intcell->chars[i];
if(!intcell->chars[i])
break;
}
cell->schar = intcell->schar;
cell->attrs.bold = intcell->pen.bold;
cell->attrs.underline = intcell->pen.underline;
@ -1007,7 +979,7 @@ int vterm_screen_get_cell(const VTermScreen *screen, VTermPos pos, VTermScreenCe
cell->uri = intcell->pen.uri;
if(pos.col < (screen->cols - 1) &&
getcell(screen, pos.row, pos.col + 1)->chars[0] == (uint32_t)-1)
getcell(screen, pos.row, pos.col + 1)->schar == (uint32_t)-1)
cell->width = 2;
else
cell->width = 1;
@ -1020,7 +992,7 @@ int vterm_screen_is_eol(const VTermScreen *screen, VTermPos pos)
/* This cell is EOL if this and every cell to the right is black */
for(; pos.col < screen->cols; pos.col++) {
ScreenCell *cell = getcell(screen, pos.row, pos.col);
if(cell->chars[0] != 0)
if(cell->schar != 0)
return 0;
}

138
src/vterm/state.c
View File

@ -3,6 +3,9 @@
#include <stdio.h>
#include <string.h>
#include "nvim/grid.h"
#include "nvim/mbyte.h"
#define strneq(a,b,n) (strncmp(a,b,n)==0)
#if defined(DEBUG) && DEBUG > 1
@ -11,10 +14,10 @@
/* Some convenient wrappers to make callback functions easier */
static void putglyph(VTermState *state, const uint32_t chars[], int width, VTermPos pos)
static void putglyph(VTermState *state, const schar_T schar, int width, VTermPos pos)
{
VTermGlyphInfo info = {
.chars = chars,
.schar = schar,
.width = width,
.protected_cell = state->protected_cell,
.dwl = state->lineinfo[pos.row].doublewidth,
@ -82,8 +85,7 @@ static VTermState *vterm_state_new(VTerm *vt)
state->bold_is_highbright = 0;
state->combine_chars_size = 16;
state->combine_chars = vterm_allocator_malloc(state->vt, state->combine_chars_size * sizeof(state->combine_chars[0]));
state->combine_pos.row = -1;
state->tabstops = vterm_allocator_malloc(state->vt, (state->cols + 7) / 8);
@ -105,7 +107,6 @@ INTERNAL void vterm_state_free(VTermState *state)
vterm_allocator_free(state->vt, state->lineinfos[BUFIDX_PRIMARY]);
if(state->lineinfos[BUFIDX_ALTSCREEN])
vterm_allocator_free(state->vt, state->lineinfos[BUFIDX_ALTSCREEN]);
vterm_allocator_free(state->vt, state->combine_chars);
vterm_allocator_free(state->vt, state);
}
@ -171,19 +172,6 @@ static void linefeed(VTermState *state)
state->pos.row++;
}
static void grow_combine_buffer(VTermState *state)
{
size_t new_size = state->combine_chars_size * 2;
uint32_t *new_chars = vterm_allocator_malloc(state->vt, new_size * sizeof(new_chars[0]));
memcpy(new_chars, state->combine_chars, state->combine_chars_size * sizeof(new_chars[0]));
vterm_allocator_free(state->vt, state->combine_chars);
state->combine_chars = new_chars;
state->combine_chars_size = new_size;
}
static void set_col_tabstop(VTermState *state, int col)
{
unsigned char mask = 1 << (col & 7);
@ -301,88 +289,35 @@ static int on_text(const char bytes[], size_t len, void *user)
state->gsingle_set = 0;
int i = 0;
GraphemeState grapheme_state = GRAPHEME_STATE_INIT;
size_t grapheme_len = 0;
bool recombine = false;
/* See if the cursor has moved since */
if(state->pos.row == state->combine_pos.row && state->pos.col == state->combine_pos.col + state->combine_width) {
/* This is a combining char. that needs to be merged with the previous
* glyph output */
if(vterm_unicode_is_combining(codepoints[i])) {
/* See if the cursor has moved since */
if(state->pos.row == state->combine_pos.row && state->pos.col == state->combine_pos.col + state->combine_width) {
#ifdef DEBUG_GLYPH_COMBINE
int printpos;
printf("DEBUG: COMBINING SPLIT GLYPH of chars {");
for(printpos = 0; state->combine_chars[printpos]; printpos++)
printf("U+%04x ", state->combine_chars[printpos]);
printf("} + {");
#endif
if(utf_iscomposing(state->grapheme_last, codepoints[i], &state->grapheme_state)) {
/* Find where we need to append these combining chars */
int saved_i = 0;
while(state->combine_chars[saved_i])
saved_i++;
/* Add extra ones */
while(i < npoints && vterm_unicode_is_combining(codepoints[i])) {
if(saved_i >= state->combine_chars_size)
grow_combine_buffer(state);
state->combine_chars[saved_i++] = codepoints[i++];
}
if(saved_i >= state->combine_chars_size)
grow_combine_buffer(state);
state->combine_chars[saved_i] = 0;
#ifdef DEBUG_GLYPH_COMBINE
for(; state->combine_chars[printpos]; printpos++)
printf("U+%04x ", state->combine_chars[printpos]);
printf("}\n");
#endif
/* Now render it */
putglyph(state, state->combine_chars, state->combine_width, state->combine_pos);
}
else {
grapheme_len = state->grapheme_len;
grapheme_state = state->grapheme_state;
state->pos.col = state->combine_pos.col;
recombine = true;
} else {
DEBUG_LOG("libvterm: TODO: Skip over split char+combining\n");
}
}
for(; i < npoints; i++) {
while(i < npoints) {
// Try to find combining characters following this
int glyph_starts = i;
int glyph_ends;
for(glyph_ends = i + 1;
(glyph_ends < npoints) && (glyph_ends < glyph_starts + VTERM_MAX_CHARS_PER_CELL);
glyph_ends++)
if(!vterm_unicode_is_combining(codepoints[glyph_ends]))
break;
int width = 0;
uint32_t chars[VTERM_MAX_CHARS_PER_CELL + 1];
for( ; i < glyph_ends; i++) {
chars[i - glyph_starts] = codepoints[i];
int this_width = vterm_unicode_width(codepoints[i]);
#ifdef DEBUG
if(this_width < 0) {
fprintf(stderr, "Text with negative-width codepoint U+%04x\n", codepoints[i]);
abort();
do {
if (grapheme_len < sizeof(state->grapheme_buf) - 4) {
grapheme_len += utf_char2bytes(codepoints[i], state->grapheme_buf + grapheme_len);
}
#endif
width += this_width;
}
while(i < npoints && vterm_unicode_is_combining(codepoints[i]))
i++;
} while(i < npoints && utf_iscomposing(codepoints[i-1], codepoints[i], &grapheme_state));
chars[glyph_ends - glyph_starts] = 0;
i--;
#ifdef DEBUG_GLYPH_COMBINE
int printpos;
printf("DEBUG: COMBINED GLYPH of %d chars {", glyph_ends - glyph_starts);
for(printpos = 0; printpos < glyph_ends - glyph_starts; printpos++)
printf("U+%04x ", chars[printpos]);
printf("}, onscreen width %d\n", width);
#endif
int width = utf_ptr2cells_len(state->grapheme_buf, grapheme_len);
if(state->at_phantom || state->pos.col + width > THISROWWIDTH(state)) {
linefeed(state);
@ -391,7 +326,7 @@ static int on_text(const char bytes[], size_t len, void *user)
state->lineinfo[state->pos.row].continuation = 1;
}
if(state->mode.insert) {
if(state->mode.insert && !recombine) {
/* TODO: This will be a little inefficient for large bodies of text, as
* it'll have to 'ICH' effectively before every glyph. We should scan
* ahead and ICH as many times as required
@ -405,22 +340,20 @@ static int on_text(const char bytes[], size_t len, void *user)
scroll(state, rect, 0, -1);
}
putglyph(state, chars, width, state->pos);
schar_T sc = schar_from_buf(state->grapheme_buf, grapheme_len);
putglyph(state, sc, width, state->pos);
if(i == npoints - 1) {
if(i == npoints) {
/* End of the buffer. Save the chars in case we have to combine with
* more on the next call */
int save_i;
for(save_i = 0; chars[save_i]; save_i++) {
if(save_i >= state->combine_chars_size)
grow_combine_buffer(state);
state->combine_chars[save_i] = chars[save_i];
}
if(save_i >= state->combine_chars_size)
grow_combine_buffer(state);
state->combine_chars[save_i] = 0;
state->grapheme_len = grapheme_len;
state->grapheme_last = codepoints[i-1];
state->grapheme_state = grapheme_state;
state->combine_width = width;
state->combine_pos = state->pos;
} else {
grapheme_len = 0;
recombine = false;
}
if(state->pos.col + width >= THISROWWIDTH(state)) {
@ -646,7 +579,7 @@ static int on_escape(const char *bytes, size_t len, void *user)
case '8': // DECALN
{
VTermPos pos;
uint32_t E[] = { 'E', 0 };
schar_T E = schar_from_ascii('E'); // E
for(pos.row = 0; pos.row < state->rows; pos.row++)
for(pos.col = 0; pos.col < ROWWIDTH(state, pos.row); pos.col++)
putglyph(state, E, 1, pos);
@ -1234,8 +1167,9 @@ static int on_csi(const char *leader, const long args[], int argcount, const cha
count = CSI_ARG_COUNT(args[0]);
col = state->pos.col + count;
UBOUND(col, row_width);
schar_T sc = schar_from_buf(state->grapheme_buf, state->grapheme_len);
while (state->pos.col < col) {
putglyph(state, state->combine_chars, state->combine_width, state->pos);
putglyph(state, sc, state->combine_width, state->pos);
state->pos.col += state->combine_width;
}
if (state->pos.col + state->combine_width >= row_width) {

313
src/vterm/unicode.c
View File

@ -1,313 +0,0 @@
#include "vterm_internal.h"
// ### The following from http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
// With modifications:
// made functions static
// moved 'combining' table to file scope, so other functions can see it
// ###################################################################
/*
* This is an implementation of wcwidth() and wcswidth() (defined in
* IEEE Std 1002.1-2001) for Unicode.
*
* http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html
* http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html
*
* In fixed-width output devices, Latin characters all occupy a single
* "cell" position of equal width, whereas ideographic CJK characters
* occupy two such cells. Interoperability between terminal-line
* applications and (teletype-style) character terminals using the
* UTF-8 encoding requires agreement on which character should advance
* the cursor by how many cell positions. No established formal
* standards exist at present on which Unicode character shall occupy
* how many cell positions on character terminals. These routines are
* a first attempt of defining such behavior based on simple rules
* applied to data provided by the Unicode Consortium.
*
* For some graphical characters, the Unicode standard explicitly
* defines a character-cell width via the definition of the East Asian
* FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes.
* In all these cases, there is no ambiguity about which width a
* terminal shall use. For characters in the East Asian Ambiguous (A)
* class, the width choice depends purely on a preference of backward
* compatibility with either historic CJK or Western practice.
* Choosing single-width for these characters is easy to justify as
* the appropriate long-term solution, as the CJK practice of
* displaying these characters as double-width comes from historic
* implementation simplicity (8-bit encoded characters were displayed
* single-width and 16-bit ones double-width, even for Greek,
* Cyrillic, etc.) and not any typographic considerations.
*
* Much less clear is the choice of width for the Not East Asian
* (Neutral) class. Existing practice does not dictate a width for any
* of these characters. It would nevertheless make sense
* typographically to allocate two character cells to characters such
* as for instance EM SPACE or VOLUME INTEGRAL, which cannot be
* represented adequately with a single-width glyph. The following
* routines at present merely assign a single-cell width to all
* neutral characters, in the interest of simplicity. This is not
* entirely satisfactory and should be reconsidered before
* establishing a formal standard in this area. At the moment, the
* decision which Not East Asian (Neutral) characters should be
* represented by double-width glyphs cannot yet be answered by
* applying a simple rule from the Unicode database content. Setting
* up a proper standard for the behavior of UTF-8 character terminals
* will require a careful analysis not only of each Unicode character,
* but also of each presentation form, something the author of these
* routines has avoided to do so far.
*
* http://www.unicode.org/unicode/reports/tr11/
*
* Markus Kuhn -- 2007-05-26 (Unicode 5.0)
*
* Permission to use, copy, modify, and distribute this software
* for any purpose and without fee is hereby granted. The author
* disclaims all warranties with regard to this software.
*
* Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
*/
struct interval {
int first;
int last;
};
/* sorted list of non-overlapping intervals of non-spacing characters */
/* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */
static const struct interval combining[] = {
{ 0x0300, 0x036F }, { 0x0483, 0x0486 }, { 0x0488, 0x0489 },
{ 0x0591, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 },
{ 0x05C4, 0x05C5 }, { 0x05C7, 0x05C7 }, { 0x0600, 0x0603 },
{ 0x0610, 0x0615 }, { 0x064B, 0x065E }, { 0x0670, 0x0670 },
{ 0x06D6, 0x06E4 }, { 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED },
{ 0x070F, 0x070F }, { 0x0711, 0x0711 }, { 0x0730, 0x074A },
{ 0x07A6, 0x07B0 }, { 0x07EB, 0x07F3 }, { 0x0901, 0x0902 },
{ 0x093C, 0x093C }, { 0x0941, 0x0948 }, { 0x094D, 0x094D },
{ 0x0951, 0x0954 }, { 0x0962, 0x0963 }, { 0x0981, 0x0981 },
{ 0x09BC, 0x09BC }, { 0x09C1, 0x09C4 }, { 0x09CD, 0x09CD },
{ 0x09E2, 0x09E3 }, { 0x0A01, 0x0A02 }, { 0x0A3C, 0x0A3C },
{ 0x0A41, 0x0A42 }, { 0x0A47, 0x0A48 }, { 0x0A4B, 0x0A4D },
{ 0x0A70, 0x0A71 }, { 0x0A81, 0x0A82 }, { 0x0ABC, 0x0ABC },
{ 0x0AC1, 0x0AC5 }, { 0x0AC7, 0x0AC8 }, { 0x0ACD, 0x0ACD },
{ 0x0AE2, 0x0AE3 }, { 0x0B01, 0x0B01 }, { 0x0B3C, 0x0B3C },
{ 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B43 }, { 0x0B4D, 0x0B4D },
{ 0x0B56, 0x0B56 }, { 0x0B82, 0x0B82 }, { 0x0BC0, 0x0BC0 },
{ 0x0BCD, 0x0BCD }, { 0x0C3E, 0x0C40 }, { 0x0C46, 0x0C48 },
{ 0x0C4A, 0x0C4D }, { 0x0C55, 0x0C56 }, { 0x0CBC, 0x0CBC },
{ 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 }, { 0x0CCC, 0x0CCD },
{ 0x0CE2, 0x0CE3 }, { 0x0D41, 0x0D43 }, { 0x0D4D, 0x0D4D },
{ 0x0DCA, 0x0DCA }, { 0x0DD2, 0x0DD4 }, { 0x0DD6, 0x0DD6 },
{ 0x0E31, 0x0E31 }, { 0x0E34, 0x0E3A }, { 0x0E47, 0x0E4E },
{ 0x0EB1, 0x0EB1 }, { 0x0EB4, 0x0EB9 }, { 0x0EBB, 0x0EBC },
{ 0x0EC8, 0x0ECD }, { 0x0F18, 0x0F19 }, { 0x0F35, 0x0F35 },
{ 0x0F37, 0x0F37 }, { 0x0F39, 0x0F39 }, { 0x0F71, 0x0F7E },
{ 0x0F80, 0x0F84 }, { 0x0F86, 0x0F87 }, { 0x0F90, 0x0F97 },
{ 0x0F99, 0x0FBC }, { 0x0FC6, 0x0FC6 }, { 0x102D, 0x1030 },
{ 0x1032, 0x1032 }, { 0x1036, 0x1037 }, { 0x1039, 0x1039 },
{ 0x1058, 0x1059 }, { 0x1160, 0x11FF }, { 0x135F, 0x135F },
{ 0x1712, 0x1714 }, { 0x1732, 0x1734 }, { 0x1752, 0x1753 },
{ 0x1772, 0x1773 }, { 0x17B4, 0x17B5 }, { 0x17B7, 0x17BD },
{ 0x17C6, 0x17C6 }, { 0x17C9, 0x17D3 }, { 0x17DD, 0x17DD },
{ 0x180B, 0x180D }, { 0x18A9, 0x18A9 }, { 0x1920, 0x1922 },
{ 0x1927, 0x1928 }, { 0x1932, 0x1932 }, { 0x1939, 0x193B },
{ 0x1A17, 0x1A18 }, { 0x1B00, 0x1B03 }, { 0x1B34, 0x1B34 },
{ 0x1B36, 0x1B3A }, { 0x1B3C, 0x1B3C }, { 0x1B42, 0x1B42 },
{ 0x1B6B, 0x1B73 }, { 0x1DC0, 0x1DCA }, { 0x1DFE, 0x1DFF },
{ 0x200B, 0x200F }, { 0x202A, 0x202E }, { 0x2060, 0x2063 },
{ 0x206A, 0x206F }, { 0x20D0, 0x20EF }, { 0x302A, 0x302F },
{ 0x3099, 0x309A }, { 0xA806, 0xA806 }, { 0xA80B, 0xA80B },
{ 0xA825, 0xA826 }, { 0xFB1E, 0xFB1E }, { 0xFE00, 0xFE0F },
{ 0xFE20, 0xFE23 }, { 0xFEFF, 0xFEFF }, { 0xFFF9, 0xFFFB },
{ 0x10A01, 0x10A03 }, { 0x10A05, 0x10A06 }, { 0x10A0C, 0x10A0F },
{ 0x10A38, 0x10A3A }, { 0x10A3F, 0x10A3F }, { 0x1D167, 0x1D169 },
{ 0x1D173, 0x1D182 }, { 0x1D185, 0x1D18B }, { 0x1D1AA, 0x1D1AD },
{ 0x1D242, 0x1D244 }, { 0xE0001, 0xE0001 }, { 0xE0020, 0xE007F },
{ 0xE0100, 0xE01EF }
};
/* auxiliary function for binary search in interval table */
static int bisearch(uint32_t ucs, const struct interval *table, int max) {
int min = 0;
int mid;
if (ucs < (uint32_t) table[0].first || ucs > (uint32_t) table[max].last)
return 0;
while (max >= min) {
mid = (min + max) / 2;
if (ucs > (uint32_t) table[mid].last)
min = mid + 1;
else if (ucs < (uint32_t) table[mid].first)
max = mid - 1;
else
return 1;
}
return 0;
}
/* The following two functions define the column width of an ISO 10646
* character as follows:
*
* - The null character (U+0000) has a column width of 0.
*
* - Other C0/C1 control characters and DEL will lead to a return
* value of -1.
*
* - Non-spacing and enclosing combining characters (general
* category code Mn or Me in the Unicode database) have a
* column width of 0.
*
* - SOFT HYPHEN (U+00AD) has a column width of 1.
*
* - Other format characters (general category code Cf in the Unicode
* database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
*
* - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
* have a column width of 0.
*
* - Spacing characters in the East Asian Wide (W) or East Asian
* Full-width (F) category as defined in Unicode Technical
* Report #11 have a column width of 2.
*
* - All remaining characters (including all printable
* ISO 8859-1 and WGL4 characters, Unicode control characters,
* etc.) have a column width of 1.
*
* This implementation assumes that uint32_t characters are encoded
* in ISO 10646.
*/
static int mk_wcwidth(uint32_t ucs)
{
/* test for 8-bit control characters */
if (ucs == 0)
return 0;
if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0))
return -1;
/* binary search in table of non-spacing characters */
if (bisearch(ucs, combining,
sizeof(combining) / sizeof(struct interval) - 1))
return 0;
/* if we arrive here, ucs is not a combining or C0/C1 control character */
return 1 +
(ucs >= 0x1100 &&
(ucs <= 0x115f || /* Hangul Jamo init. consonants */
ucs == 0x2329 || ucs == 0x232a ||
(ucs >= 0x2e80 && ucs <= 0xa4cf &&
ucs != 0x303f) || /* CJK ... Yi */
(ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */
(ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */
(ucs >= 0xfe10 && ucs <= 0xfe19) || /* Vertical forms */
(ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */
(ucs >= 0xff00 && ucs <= 0xff60) || /* Fullwidth Forms */
(ucs >= 0xffe0 && ucs <= 0xffe6) ||
(ucs >= 0x20000 && ucs <= 0x2fffd) ||
(ucs >= 0x30000 && ucs <= 0x3fffd)));
}
#ifdef USE_MK_WCWIDTH_CJK
/*
* The following functions are the same as mk_wcwidth() and
* mk_wcswidth(), except that spacing characters in the East Asian
* Ambiguous (A) category as defined in Unicode Technical Report #11
* have a column width of 2. This variant might be useful for users of
* CJK legacy encodings who want to migrate to UCS without changing
* the traditional terminal character-width behaviour. It is not
* otherwise recommended for general use.
*/
static int mk_wcwidth_cjk(uint32_t ucs)
{
/* sorted list of non-overlapping intervals of East Asian Ambiguous
* characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */
static const struct interval ambiguous[] = {
{ 0x00A1, 0x00A1 }, { 0x00A4, 0x00A4 }, { 0x00A7, 0x00A8 },
{ 0x00AA, 0x00AA }, { 0x00AE, 0x00AE }, { 0x00B0, 0x00B4 },
{ 0x00B6, 0x00BA }, { 0x00BC, 0x00BF }, { 0x00C6, 0x00C6 },
{ 0x00D0, 0x00D0 }, { 0x00D7, 0x00D8 }, { 0x00DE, 0x00E1 },
{ 0x00E6, 0x00E6 }, { 0x00E8, 0x00EA }, { 0x00EC, 0x00ED },
{ 0x00F0, 0x00F0 }, { 0x00F2, 0x00F3 }, { 0x00F7, 0x00FA },
{ 0x00FC, 0x00FC }, { 0x00FE, 0x00FE }, { 0x0101, 0x0101 },
{ 0x0111, 0x0111 }, { 0x0113, 0x0113 }, { 0x011B, 0x011B },
{ 0x0126, 0x0127 }, { 0x012B, 0x012B }, { 0x0131, 0x0133 },
{ 0x0138, 0x0138 }, { 0x013F, 0x0142 }, { 0x0144, 0x0144 },
{ 0x0148, 0x014B }, { 0x014D, 0x014D }, { 0x0152, 0x0153 },
{ 0x0166, 0x0167 }, { 0x016B, 0x016B }, { 0x01CE, 0x01CE },
{ 0x01D0, 0x01D0 }, { 0x01D2, 0x01D2 }, { 0x01D4, 0x01D4 },
{ 0x01D6, 0x01D6 }, { 0x01D8, 0x01D8 }, { 0x01DA, 0x01DA },
{ 0x01DC, 0x01DC }, { 0x0251, 0x0251 }, { 0x0261, 0x0261 },
{ 0x02C4, 0x02C4 }, { 0x02C7, 0x02C7 }, { 0x02C9, 0x02CB },
{ 0x02CD, 0x02CD }, { 0x02D0, 0x02D0 }, { 0x02D8, 0x02DB },
{ 0x02DD, 0x02DD }, { 0x02DF, 0x02DF }, { 0x0391, 0x03A1 },
{ 0x03A3, 0x03A9 }, { 0x03B1, 0x03C1 }, { 0x03C3, 0x03C9 },
{ 0x0401, 0x0401 }, { 0x0410, 0x044F }, { 0x0451, 0x0451 },
{ 0x2010, 0x2010 }, { 0x2013, 0x2016 }, { 0x2018, 0x2019 },
{ 0x201C, 0x201D }, { 0x2020, 0x2022 }, { 0x2024, 0x2027 },
{ 0x2030, 0x2030 }, { 0x2032, 0x2033 }, { 0x2035, 0x2035 },
{ 0x203B, 0x203B }, { 0x203E, 0x203E }, { 0x2074, 0x2074 },
{ 0x207F, 0x207F }, { 0x2081, 0x2084 }, { 0x20AC, 0x20AC },
{ 0x2103, 0x2103 }, { 0x2105, 0x2105 }, { 0x2109, 0x2109 },
{ 0x2113, 0x2113 }, { 0x2116, 0x2116 }, { 0x2121, 0x2122 },
{ 0x2126, 0x2126 }, { 0x212B, 0x212B }, { 0x2153, 0x2154 },
{ 0x215B, 0x215E }, { 0x2160, 0x216B }, { 0x2170, 0x2179 },
{ 0x2190, 0x2199 }, { 0x21B8, 0x21B9 }, { 0x21D2, 0x21D2 },
{ 0x21D4, 0x21D4 }, { 0x21E7, 0x21E7 }, { 0x2200, 0x2200 },
{ 0x2202, 0x2203 }, { 0x2207, 0x2208 }, { 0x220B, 0x220B },
{ 0x220F, 0x220F }, { 0x2211, 0x2211 }, { 0x2215, 0x2215 },
{ 0x221A, 0x221A }, { 0x221D, 0x2220 }, { 0x2223, 0x2223 },
{ 0x2225, 0x2225 }, { 0x2227, 0x222C }, { 0x222E, 0x222E },
{ 0x2234, 0x2237 }, { 0x223C, 0x223D }, { 0x2248, 0x2248 },
{ 0x224C, 0x224C }, { 0x2252, 0x2252 }, { 0x2260, 0x2261 },
{ 0x2264, 0x2267 }, { 0x226A, 0x226B }, { 0x226E, 0x226F },
{ 0x2282, 0x2283 }, { 0x2286, 0x2287 }, { 0x2295, 0x2295 },
{ 0x2299, 0x2299 }, { 0x22A5, 0x22A5 }, { 0x22BF, 0x22BF },
{ 0x2312, 0x2312 }, { 0x2460, 0x24E9 }, { 0x24EB, 0x254B },
{ 0x2550, 0x2573 }, { 0x2580, 0x258F }, { 0x2592, 0x2595 },
{ 0x25A0, 0x25A1 }, { 0x25A3, 0x25A9 }, { 0x25B2, 0x25B3 },
{ 0x25B6, 0x25B7 }, { 0x25BC, 0x25BD }, { 0x25C0, 0x25C1 },
{ 0x25C6, 0x25C8 }, { 0x25CB, 0x25CB }, { 0x25CE, 0x25D1 },
{ 0x25E2, 0x25E5 }, { 0x25EF, 0x25EF }, { 0x2605, 0x2606 },
{ 0x2609, 0x2609 }, { 0x260E, 0x260F }, { 0x2614, 0x2615 },
{ 0x261C, 0x261C }, { 0x261E, 0x261E }, { 0x2640, 0x2640 },
{ 0x2642, 0x2642 }, { 0x2660, 0x2661 }, { 0x2663, 0x2665 },
{ 0x2667, 0x266A }, { 0x266C, 0x266D }, { 0x266F, 0x266F },
{ 0x273D, 0x273D }, { 0x2776, 0x277F }, { 0xE000, 0xF8FF },
{ 0xFFFD, 0xFFFD }, { 0xF0000, 0xFFFFD }, { 0x100000, 0x10FFFD }
};
/* binary search in table of non-spacing characters */
if (bisearch(ucs, ambiguous,
sizeof(ambiguous) / sizeof(struct interval) - 1))
return 2;
return mk_wcwidth(ucs);
}
#endif
// ################################
// ### The rest added by Paul Evans
static const struct interval fullwidth[] = {
#include "fullwidth.inc"
};
INTERNAL int vterm_unicode_width(uint32_t codepoint)
{
if(bisearch(codepoint, fullwidth, sizeof(fullwidth) / sizeof(fullwidth[0]) - 1))
return 2;
return mk_wcwidth(codepoint);
}
INTERNAL int vterm_unicode_is_combining(uint32_t codepoint)
{
return bisearch(codepoint, combining, sizeof(combining) / sizeof(struct interval) - 1);
}

10
src/vterm/vterm.h
View File

@ -10,6 +10,7 @@ extern "C" {
#include <stdbool.h>
#include "nvim/macros_defs.h"
#include "nvim/types_defs.h"
#include "vterm_keycodes.h"
#define VTERM_VERSION_MAJOR 0
@ -19,11 +20,6 @@ extern "C" {
#define VTERM_CHECK_VERSION \
vterm_check_version(VTERM_VERSION_MAJOR, VTERM_VERSION_MINOR)
/* Any cell can contain at most one basic printing character and 5 combining
* characters. This number could be changed but will be ABI-incompatible if
* you do */
enum{ VTERM_MAX_CHARS_PER_CELL=6};
typedef struct VTerm VTerm;
typedef struct VTermState VTermState;
typedef struct VTermScreen VTermScreen;
@ -292,7 +288,7 @@ typedef enum {
} VTermSelectionMask;
typedef struct {
const uint32_t *chars;
schar_T schar;
int width;
unsigned int protected_cell:1; /* DECSCA-protected against DECSEL/DECSED */
unsigned int dwl:1; /* DECDWL or DECDHL double-width line */
@ -528,7 +524,7 @@ enum {
};
typedef struct {
uint32_t chars[VTERM_MAX_CHARS_PER_CELL];
schar_T schar;
char width;
VTermScreenCellAttrs attrs;
VTermColor fg, bg;

10
src/vterm/vterm_internal.h
View File

@ -4,6 +4,7 @@
#include "vterm.h"
#include <stdarg.h>
#include "nvim/mbyte.h"
#if defined(__GNUC__)
# define INTERNAL __attribute__((visibility("internal")))
@ -101,8 +102,10 @@ struct VTermState
enum { MOUSE_X10, MOUSE_UTF8, MOUSE_SGR, MOUSE_RXVT } mouse_protocol;
/* Last glyph output, for Unicode recombining purposes */
uint32_t *combine_chars;
size_t combine_chars_size; // Number of ELEMENTS in the above
char grapheme_buf[MAX_SCHAR_SIZE];
size_t grapheme_len;
uint32_t grapheme_last; // last added UTF-32 char
GraphemeState grapheme_state;
int combine_width; // The width of the glyph above
VTermPos combine_pos; // Position before movement
@ -292,7 +295,4 @@ void vterm_screen_free(VTermScreen *screen);
VTermEncoding *vterm_lookup_encoding(VTermEncodingType type, char designation);
int vterm_unicode_width(uint32_t codepoint);
int vterm_unicode_is_combining(uint32_t codepoint);
#endif

19
test/functional/terminal/buffer_spec.lua
View File

@ -400,15 +400,28 @@ describe(':terminal buffer', function()
assert_alive()
end)
it('truncates number of composing characters to 5', function()
it('truncates the size of grapheme clusters', function()
local chan = api.nvim_open_term(0, {})
local composing = ('a̳'):sub(2)
api.nvim_chan_send(chan, 'a' .. composing:rep(8))
api.nvim_chan_send(chan, 'a' .. composing:rep(20))
retry(nil, nil, function()
eq('a' .. composing:rep(5), api.nvim_get_current_line())
eq('a' .. composing:rep(14), api.nvim_get_current_line())
end)
end)
it('handles extended grapheme clusters', function()
local screen = Screen.new(50, 7)
feed 'i'
local chan = api.nvim_open_term(0, {})
api.nvim_chan_send(chan, '🏴‍☠️ yarrr')
screen:expect([[
🏴‍☠️ yarrr^ |
|*5
{5:-- TERMINAL --} |
]])
eq('🏴‍☠️ yarrr', api.nvim_get_current_line())
end)
it('handles split UTF-8 sequences #16245', function()
local screen = Screen.new(50, 7)
fn.jobstart({ testprg('shell-test'), 'UTF-8' }, { term = true })

37
test/unit/fixtures/vterm_test.c
View File

@ -1,4 +1,6 @@
#include <stdio.h>
#include "nvim/grid.h"
#include "nvim/mbyte.h"
#include "vterm_test.h"
@ -202,6 +204,26 @@ int selection_query(VTermSelectionMask mask, void *user)
return 1;
}
static void print_schar(FILE *f, schar_T schar) {
char buf[MAX_SCHAR_SIZE];
schar_get(buf, schar);
StrCharInfo ci = utf_ptr2StrCharInfo(buf);
bool did = false;
while (*ci.ptr != 0) {
if (did) {
fprintf(f, ",");
}
if (ci.chr.len == 1 && ci.chr.value >= 0x80) {
fprintf(f, "??%x", ci.chr.value);
} else {
fprintf(f, "%x", ci.chr.value);
}
did = true;
ci = utf_ptr2StrCharInfo(ci.ptr + ci.chr.len);
}
}
bool want_state_putglyph;
int state_putglyph(VTermGlyphInfo *info, VTermPos pos, void *user)
{
@ -211,9 +233,7 @@ int state_putglyph(VTermGlyphInfo *info, VTermPos pos, void *user)
FILE *f = fopen(VTERM_TEST_FILE, "a");
fprintf(f, "putglyph ");
for (int i = 0; i < VTERM_MAX_CHARS_PER_CELL && info->chars[i]; i++) {
fprintf(f, i ? ",%x" : "%x", info->chars[i]);
}
print_schar(f, info->schar);
fprintf(f, " %d %d,%d", info->width, pos.row, pos.col);
if (info->protected_cell) {
fprintf(f, " prot");
@ -443,14 +463,15 @@ int screen_sb_pushline(int cols, const VTermScreenCell *cells, void *user)
}
int eol = cols;
while (eol && !cells[eol - 1].chars[0]) {
while (eol && !cells[eol-1].schar) {
eol--;
}
FILE *f = fopen(VTERM_TEST_FILE, "a");
fprintf(f, "sb_pushline %d =", cols);
for (int c = 0; c < eol; c++) {
fprintf(f, " %02X", cells[c].chars[0]);
fprintf(f, " ");
print_schar(f, cells[c].schar);
}
fprintf(f, "\n");
@ -467,10 +488,10 @@ int screen_sb_popline(int cols, VTermScreenCell *cells, void *user)
// All lines of scrollback contain "ABCDE"
for (int col = 0; col < cols; col++) {
if (col < 5) {
cells[col].chars[0] = (uint32_t)('A' + col);
if(col < 5) {
cells[col].schar = schar_from_ascii((uint32_t)('A' + col));
} else {
cells[col].chars[0] = 0;
cells[col].schar = 0;
}
cells[col].width = 1;

83
test/unit/vterm_spec.lua
View File

@ -17,7 +17,6 @@ local bit = require('bit')
--- @field VTERM_KEY_NONE integer
--- @field VTERM_KEY_TAB integer
--- @field VTERM_KEY_UP integer
--- @field VTERM_MAX_CHARS_PER_CELL integer
--- @field VTERM_MOD_ALT integer
--- @field VTERM_MOD_CTRL integer
--- @field VTERM_MOD_SHIFT integer
@ -80,6 +79,8 @@ local bit = require('bit')
--- @field vterm_state_set_selection_callbacks function
--- @field vterm_state_set_unrecognised_fallbacks function
local vterm = t.cimport(
'./src/nvim/mbyte.h',
'./src/nvim/grid.h',
'./src/vterm/vterm.h',
'./src/vterm/vterm_internal.h',
'./test/unit/fixtures/vterm_test.h'
@ -302,16 +303,12 @@ local function screen_chars(start_row, start_col, end_row, end_col, expected, sc
rect['end_row'] = end_row
rect['end_col'] = end_col
local len = vterm.vterm_screen_get_chars(screen, nil, 0, rect)
local len = vterm.vterm_screen_get_text(screen, nil, 0, rect)
local chars = t.ffi.new('uint32_t[?]', len)
vterm.vterm_screen_get_chars(screen, chars, len, rect)
local actual = ''
for i = 0, tonumber(len) - 1 do
actual = actual .. string.char(chars[i])
end
local text = t.ffi.new('unsigned char[?]', len)
vterm.vterm_screen_get_text(screen, text, len, rect)
local actual = t.ffi.string(text, len)
t.eq(expected, actual)
end
@ -349,7 +346,7 @@ local function screen_row(row, expected, screen, end_col)
local text = t.ffi.new('unsigned char[?]', len)
vterm.vterm_screen_get_text(screen, text, len, rect)
t.eq(expected, t.ffi.string(text))
t.eq(expected, t.ffi.string(text, len))
end
local function screen_cell(row, col, expected, screen)
@ -360,14 +357,20 @@ local function screen_cell(row, col, expected, screen)
local cell = t.ffi.new('VTermScreenCell')
vterm.vterm_screen_get_cell(screen, pos, cell)
local buf = t.ffi.new('unsigned char[32]')
vterm.schar_get(buf, cell.schar)
local actual = '{'
for i = 0, vterm.VTERM_MAX_CHARS_PER_CELL - 1 do
if cell['chars'][i] ~= 0 then
if i > 0 then
actual = actual .. ','
end
actual = string.format('%s%02x', actual, cell['chars'][i])
local i = 0
while buf[i] > 0 do
local char = vterm.utf_ptr2char(buf + i)
local charlen = vterm.utf_ptr2len(buf + i)
if i > 0 then
actual = actual .. ','
end
local invalid = char >= 128 and charlen == 1
actual = string.format('%s%s%02x', actual, invalid and '?' or '', char)
i = i + charlen
end
actual = string.format('%s} width=%d attrs={', actual, cell['width'])
actual = actual .. (cell['attrs'].bold ~= 0 and 'B' or '')
@ -962,8 +965,8 @@ describe('vterm', function()
-- Spare combining chars get truncated
reset(state, nil)
push('e' .. string.rep('\xCC\x81', 10), vt)
expect('putglyph 65,301,301,301,301,301 1 0,0') -- and nothing more
push('e' .. string.rep('\xCC\x81', 20), vt)
expect('putglyph 65,301,301,301,301,301,301,301,301,301,301,301,301,301,301 1 0,0') -- and nothing more
reset(state, nil)
push('e', vt)
@ -973,6 +976,34 @@ describe('vterm', function()
push('\xCC\x82', vt)
expect('putglyph 65,301,302 1 0,0')
-- emoji with ZWJ and variant selectors, as one chunk
reset(state, nil)
push('🏳️‍🌈🏳️‍⚧️🏴‍☠️', vt)
expect([[putglyph 1f3f3,fe0f,200d,1f308 2 0,0
putglyph 1f3f3,fe0f,200d,26a7,fe0f 2 0,2
putglyph 1f3f4,200d,2620,fe0f 2 0,4]])
-- emoji, one code point at a time
reset(state, nil)
push('🏳', vt)
expect('putglyph 1f3f3 2 0,0')
push('\xef\xb8\x8f', vt)
expect('putglyph 1f3f3,fe0f 2 0,0')
push('\xe2\x80\x8d', vt)
expect('putglyph 1f3f3,fe0f,200d 2 0,0')
push('🌈', vt)
expect('putglyph 1f3f3,fe0f,200d,1f308 2 0,0')
-- modifier can change width
push('❤', vt)
expect('putglyph 2764 1 0,2')
push('\xef\xb8\x8f', vt)
expect('putglyph 2764,fe0f 2 0,2')
-- also works batched
push('❤️', vt)
expect('putglyph 2764,fe0f 2 0,4')
-- DECSCA protected
reset(state, nil)
push('A\x1b[1"qB\x1b[2"qC', vt)
@ -3046,7 +3077,7 @@ describe('vterm', function()
screen_cell(
0,
0,
'{65,301,302,303,304,305} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)',
'{65,301,302,303,304,305,306,307,308,309,30a} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)',
screen
)
@ -3063,7 +3094,7 @@ describe('vterm', function()
screen_cell(
0,
0,
'{65,301,301,301,301,301} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)',
'{65,301,301,301,301,301,301,301,301,301,301,301,301,301,301} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)',
screen
)
@ -3072,6 +3103,16 @@ describe('vterm', function()
push('\x1b[80G\xEF\xBC\x90', vt)
screen_cell(0, 79, '{} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)', screen)
screen_cell(1, 0, '{ff10} width=2 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)', screen)
-- Outputting emoji with ZWJ and variant selectors
reset(nil, screen)
push('🏳️‍🌈🏳️‍⚧️🏴‍☠️', vt)
-- stylua: ignore start
screen_cell(0, 0, '{1f3f3,fe0f,200d,1f308} width=2 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)', screen)
screen_cell(0, 2, '{1f3f3,fe0f,200d,26a7,fe0f} width=2 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)', screen)
screen_cell(0, 4, '{1f3f4,200d,2620,fe0f} width=2 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)', screen)
-- stylua: ignore end
end)
pending('62screen_damage', function() end)
@ -3125,7 +3166,7 @@ describe('vterm', function()
screen = wantscreen(vt, { b = true })
resize(20, 80, vt)
expect(
'sb_pushline 80 = 54 6F 70\nsb_pushline 80 =\nsb_pushline 80 =\nsb_pushline 80 =\nsb_pushline 80 ='
'sb_pushline 80 = 54 6f 70\nsb_pushline 80 =\nsb_pushline 80 =\nsb_pushline 80 =\nsb_pushline 80 ='
)
-- TODO(dundargoc): fix or remove
-- screen_row( 0 , "",screen)