diff --git a/Filelist b/Filelist index 34b1a239c6..10cddd00b8 100644 --- a/Filelist +++ b/Filelist @@ -878,6 +878,7 @@ RT_SCRIPTS = \ runtime/syntax/testdir/input/setup/*.* \ runtime/syntax/testdir/dumps/*.dump \ runtime/syntax/testdir/dumps/*.vim \ + runtime/syntax/testdir/tools/* \ runtime/syntax/generator/Makefile \ runtime/syntax/generator/README.md \ runtime/syntax/generator/gen_syntax_vim.vim \ diff --git a/runtime/syntax/Makefile b/runtime/syntax/Makefile index 84d6f83dea..e981ed01a8 100644 --- a/runtime/syntax/Makefile +++ b/runtime/syntax/Makefile @@ -3,7 +3,7 @@ # To run the test manually: # ../../src/vim -u 'testdir/runtest.vim' --cmd 'breakadd func RunTest' -# Override this if needed, the default assumes Vim was build in the src dir. +# Override this if needed, the default assumes Vim was built in the src dir. #VIMPROG = vim VIMPROG = ../../src/vim @@ -13,6 +13,10 @@ VIMRUNTIME = ../.. # Uncomment this line to use valgrind for memory leaks and extra warnings. # VALGRIND = valgrind --tool=memcheck --leak-check=yes --num-callers=45 --log-file=valgrind.$* +# Trace ruler liveness on demand. +# VIM_SYNTAX_TEST_LOG = `pwd`/testdir/failed/00-TRACE_LOG + +# ENVVARS = LC_ALL=C VIM_SYNTAX_TEST_LOG="$(VIM_SYNTAX_TEST_LOG)" # ENVVARS = LC_ALL=C LANG=C LANGUAGE=C # Run the syntax tests with a C locale ENVVARS = LC_ALL=C @@ -31,6 +35,9 @@ test: @# the "vimcmd" file is used by the screendump utils @echo "../$(VIMPROG)" > testdir/vimcmd @echo "$(RUN_VIMTEST)" >> testdir/vimcmd + @# Trace ruler liveness on demand. + @#mkdir -p testdir/failed + @#touch "$(VIM_SYNTAX_TEST_LOG)" VIMRUNTIME=$(VIMRUNTIME) $(ENVVARS) $(VIMPROG) --clean --not-a-term $(DEBUGLOG) -u testdir/runtest.vim > /dev/null @rm -f testdir/Xfilter @# FIXME: Temporarily show the whole file to find out what goes wrong diff --git a/runtime/syntax/testdir/runtest.vim b/runtime/syntax/testdir/runtest.vim index 7113602b67..34b5caee91 100644 --- a/runtime/syntax/testdir/runtest.vim +++ b/runtime/syntax/testdir/runtest.vim @@ -15,6 +15,17 @@ let s:messagesFname = fnameescape(syntaxDir .. '/testdir/messages') let s:messages = [] +" Erase the cursor line and do not advance the cursor. +def EraseLineAndReturnCarriage(rname: string) + const full_width: number = winwidth(0) + const half_width: number = full_width - (full_width + 1) / 2 + if (strlen(rname) + strlen('Test' .. "\x20\x20" .. 'FAILED')) > half_width + echon "\r" .. repeat("\x20", full_width) .. "\r" + else + echon repeat("\x20", half_width) .. "\r" + endif +enddef + " Add one message to the list of messages func Message(msg) echomsg a:msg @@ -30,22 +41,23 @@ endfunc " Append s:messages to the messages file and make it empty. func AppendMessages(header) - exe 'split ' .. s:messagesFname + silent exe 'split ' .. s:messagesFname call append(line('$'), '') call append(line('$'), a:header) call append(line('$'), s:messages) let s:messages = [] - wq + silent wq endfunc " Relevant messages are written to the "messages" file. " If the file already exists it is appended to. -exe 'split ' .. s:messagesFname +silent exe 'split ' .. s:messagesFname call append(line('$'), repeat('=-', 70)) call append(line('$'), '') let s:test_run_message = 'Test run on ' .. strftime("%Y %b %d %H:%M:%S") call append(line('$'), s:test_run_message) -wq +silent wq +echo "\n" if syntaxDir !~ '[/\\]runtime[/\\]syntax\>' call Fatal('Current directory must be "runtime/syntax"') @@ -88,26 +100,127 @@ func HandleSwapExists() endif endfunc -def IsWinNumOneAtEOF(in_name_and_out_name: string): bool - # Expect defaults from term_util#RunVimInTerminal(). +" Trace ruler liveness on demand. +if !empty($VIM_SYNTAX_TEST_LOG) && filewritable($VIM_SYNTAX_TEST_LOG) + def s:TraceRulerLiveness(context: string, times: number, tail: string) + writefile([printf('%s: %4d: %s', context, times, tail)], + $VIM_SYNTAX_TEST_LOG, + 'a') + enddef +else + def s:TraceRulerLiveness(_: string, _: number, _: string) + enddef +endif + +" See ":help 'ruler'". +def s:CannotSeeLastLine(ruler: list): bool + return !(get(ruler, -1, '') ==# 'All' || get(ruler, -1, '') ==# 'Bot') +enddef + +def s:CannotDumpNextPage(buf: number, prev_ruler: list, ruler: list): bool + return !(ruler !=# prev_ruler && + len(ruler) == 2 && + ruler[1] =~# '\%(\d%\|\, ruler: list): bool + return !(len(ruler) == 2 && + ruler[1] =~# '\%(\, ruler: list): bool + return !(len(ruler) > 3 && + get(ruler, -1, '') =~# '\%(\ + # Expect defaults from "term_util#RunVimInTerminal()". if winwidth(1) != 75 || winheight(1) != 20 ch_log(printf('Aborting for %s: (75 x 20) != (%d x %d)', in_name_and_out_name, winwidth(1), winheight(1))) - return true + return ['0,0-1', 'All'] endif - # A two-fold role: (1) redraw whenever the first test file is of 19 lines or - # less long (not applicable to c.c); (2) redraw in case the terminal buffer - # cannot redraw itself just yet (else expect extra files generated). + # A two-fold role for redrawing: + # (*) in case the terminal buffer cannot redraw itself just yet; + # (*) to avoid extra "real estate" checks. redraw - const pos: string = join([ - screenstring(20, 71), - screenstring(20, 72), - screenstring(20, 73), - screenstring(20, 74), - screenstring(20, 75)], '') - return (pos == ' All ' || pos == ' Bot ') + # The contents of "ruler". + var ruler: list = [] + # Attempts at most, targeting ASan-instrumented Vim builds. + var times: number = 2048 + # Check "real estate" of the terminal buffer. Read and compare its ruler + # line and let "Xtestscript#s:AssertCursorForwardProgress()" do the rest. + # Note that the cursor ought to be advanced after each successive call of + # this function yet its relative position need not be changed (e.g. "0%"). + while CannotDumpPage(ruler) && times > 0 + ruler = split(term_getline(buf, 20)) + sleep 1m + times -= 1 + if times % 8 == 0 + redraw + endif + endwhile + TraceRulerLiveness('P', (2048 - times), in_name_and_out_name) + return ruler +enddef + +" Prevent "s:TermPollRuler()" from prematurely reading the cursor position, +" which is available at ":edit", after outracing the loading of syntax etc. in +" the terminal buffer. (Call the function before calling "VerifyScreenDump()" +" for the first time.) +def s:TermWaitAndPollRuler(buf: number, in_name_and_out_name: string): list + # Expect defaults from "term_util#RunVimInTerminal()". + if winwidth(1) != 75 || winheight(1) != 20 + ch_log(printf('Aborting for %s: (75 x 20) != (%d x %d)', + in_name_and_out_name, + winwidth(1), + winheight(1))) + return ['0,0-1', 'All'] + endif + # The contents of "ruler". + var ruler: string = '' + # Attempts at most, targeting ASan-instrumented Vim builds. + var times: number = 32768 + # Check "real estate" of the terminal buffer. Expect a known token to be + # rendered in the terminal buffer; its prefix must be "is_" so that buffer + # variables from "sh.vim" can be matched (see "Xtestscript#ShellInfo()"). + # Verify that the whole line is available! + while ruler !~# '^is_.\+\s\%(All\|Top\)$' && times > 0 + ruler = term_getline(buf, 20) + sleep 1m + times -= 1 + if times % 16 == 0 + redraw + endif + endwhile + TraceRulerLiveness('W', (32768 - times), in_name_and_out_name) + if strpart(ruler, 0, 8) !=# 'is_nonce' + # Retain any of "b:is_(bash|dash|kornshell|posix|sh)" entries and let + # "CannotDumpShellFirstPage()" win the cursor race. + return TermPollRuler( + function(CannotDumpShellFirstPage, [buf, []]), + buf, + in_name_and_out_name) + else + # Clear the "is_nonce" token and let "CannotDumpFirstPage()" win any + # race. + term_sendkeys(buf, ":redraw!\") + endif + return TermPollRuler( + function(CannotDumpFirstPage, [buf, []]), + buf, + in_name_and_out_name) enddef func RunTest() @@ -337,41 +450,44 @@ func RunTest() " load filetype specific settings call term_sendkeys(buf, ":call LoadFiletype('" .. filetype .. "')\") + " Make a synchronisation point between buffers by requesting to echo + " a known token in the terminal buffer and asserting its availability + " with "s:TermWaitAndPollRuler()". if filetype == 'sh' call term_sendkeys(buf, ":call ShellInfo()\") + else + call term_sendkeys(buf, ":echo 'is_nonce'\") endif - " Screendump at the start of the file: failed/root_00.dump let root_00 = root .. '_00' let in_name_and_out_name = fname .. ': failed/' .. root_00 .. '.dump' + " Queue up all "term_sendkeys()"es and let them finish before returning + " from "s:TermWaitAndPollRuler()". + let ruler = s:TermWaitAndPollRuler(buf, in_name_and_out_name) call ch_log('First screendump for ' .. in_name_and_out_name) + " Make a screendump at the start of the file: failed/root_00.dump let fail = VerifyScreenDump(buf, root_00, {}) - " Make a Screendump every 18 lines of the file: failed/root_NN.dump - let nr = 1 - let root_next = printf('%s_%02d', root, nr) - let in_name_and_out_name = fname .. ': failed/' .. root_next .. '.dump' - " Accommodate the next code block to "buf"'s contingency for self " wipe-out. try - if !IsWinNumOneAtEOF(in_name_and_out_name) - call term_sendkeys(buf, ":call ScrollToSecondPage((18 * 75 + 1), 19, 5) | redraw!\") - call ch_log('Next screendump for ' .. in_name_and_out_name) - let fail += VerifyScreenDump(buf, root_next, {}) + let nr = 0 + let keys_a = ":call ScrollToSecondPage((18 * 75 + 1), 19, 5) | redraw!\" + let keys_b = ":call ScrollToNextPage((18 * 75 + 1), 19, 5) | redraw!\" + while s:CannotSeeLastLine(ruler) + call term_sendkeys(buf, keys_a) + let keys_a = keys_b let nr += 1 let root_next = printf('%s_%02d', root, nr) let in_name_and_out_name = fname .. ': failed/' .. root_next .. '.dump' - - while !IsWinNumOneAtEOF(in_name_and_out_name) - call term_sendkeys(buf, ":call ScrollToNextPage((18 * 75 + 1), 19, 5) | redraw!\") - call ch_log('Next screendump for ' .. in_name_and_out_name) - let fail += VerifyScreenDump(buf, root_next, {}) - let nr += 1 - let root_next = printf('%s_%02d', root, nr) - let in_name_and_out_name = fname .. ': failed/' .. root_next .. '.dump' - endwhile - endif + let ruler = s:TermPollRuler( + \ function('s:CannotDumpNextPage', [buf, ruler]), + \ buf, + \ in_name_and_out_name) + call ch_log('Next screendump for ' .. in_name_and_out_name) + " Make a screendump of every 18 lines of the file: failed/root_NN.dump + let fail += VerifyScreenDump(buf, root_next, {}) + endwhile call StopVimInTerminal(buf) finally call delete('Xtestscript') @@ -413,6 +529,8 @@ func RunTest() let skipped_count += 1 endif + call EraseLineAndReturnCarriage(root) + " Append messages to the file "testdir/messages" call AppendMessages('Input file ' .. fname .. ':') @@ -421,6 +539,7 @@ func RunTest() endif endfor + call EraseLineAndReturnCarriage('') call Message(s:test_run_message) call Message('OK: ' .. ok_count) call Message('FAILED: ' .. len(failed_tests) .. ': ' .. string(failed_tests)) @@ -446,4 +565,4 @@ endif qall! -" vim:ts=8 +" vim:sw=2:ts=8:noet: diff --git a/runtime/syntax/testdir/tools/regenerate_screendumps.sh b/runtime/syntax/testdir/tools/regenerate_screendumps.sh new file mode 100755 index 0000000000..f85252a32a --- /dev/null +++ b/runtime/syntax/testdir/tools/regenerate_screendumps.sh @@ -0,0 +1,126 @@ +#!/bin/sh -e +# +# The following steps are to be taken by this script: +# 1) Remove all files from the "dumps" directory. +# 2) Generate screendumps for each syntax test and each self-test. +# 3) Unconditionally move each batch of screendumps to "dumps"; if generated +# files differ on repeated runs, always remove these files from "dumps". +# 4) Repeat steps 2) and 3) once or as many times as requested with the "$1" +# argument. +# 5) Summarise any differences. +# +# Provided that "git difftool" is set up (see src/testdir/commondumps.vim), +# run "git difftool HEAD -- '**/*.dump'" to collate tracked and generated +# screendumps. + +case "$1" in +-h | --help) + printf >&2 "Usage: [time VIM_SYNTAX_TEST_LOG=/tmp/log] $0 [1 | 2 | ...]\n" + exit 0 + ;; +esac + +tries="${1:-1}" +shift $# + +case "$tries" in +0* | *[!0-9]*) + exit 80 + ;; +esac + +test -x "$(command -v make)" || exit 81 +test -x "$(command -v git)" || exit 82 + +case "$(git status --porcelain=v1)" in +'') ;; +*) printf >&2 'Resolve ALL changes before proceeding.\n' + exit 83 + ;; +esac + +templet=$(printf "\t\t\t\t$(tput rev)%%s$(tput sgr0)") || exit 84 +cd "$(dirname "$0")/../../../syntax" || exit 85 +set +f +rm testdir/dumps/*.dump || exit 86 +spuriosities='' + +# Because the clean target of Make will be executed before each syntax test, +# this environment variable needs to be pointed to an existing file that is +# created in a directory not affectable by the target. +if test -w "$VIM_SYNTAX_TEST_LOG" +then + log=-e VIM_SYNTAX_TEST_LOG="$VIM_SYNTAX_TEST_LOG" +else + log= +fi + +for f in testdir/input/*.* +do + test ! -d "$f" || continue + b=$(basename "$f") + i=0 + printf "$templet\n\n" "$b" + + while test "$i" -le "$tries" + do + make $log clean "$b" test || : + + case "$i" in + 0) mv testdir/failed/*.dump testdir/dumps/ + ;; + *) case "$(printf '%s' testdir/failed/*.dump)" in + testdir/failed/\*.dump) + # (Repeatable) success. + ;; + *) spuriosities="${spuriosities}${b} " + p=${b%.*} + rm -f testdir/dumps/"$p"_[0-9][0-9].dump \ + testdir/dumps/"$p"_[0-9][0-9][0-9].dump \ + testdir/dumps/"$p"_[0-9][0-9][0-9][0-9].dump + ;; + esac + ;; + esac + + i=$(($i + 1)) + sleep 1 + done +done + +# For a 20-file set, initially fail for a series of: 1-6, 7-12, 13-18, 19-20. +tries=$(($tries + 3)) +i=0 + +while test "$i" -le "$tries" +do + make $log clean self-testing test || : + + case "$i" in + [0-3]) mv testdir/failed/dots_*.dump testdir/dumps/ + ;; + *) case "$(printf '%s' testdir/failed/*.dump)" in + testdir/failed/\*.dump) + # (Repeatable) success. + ;; + *) spuriosities="${spuriosities}dots_xy " + rm -f testdir/dumps/dots_*.dump + ;; + esac + ;; + esac + + sleep 1 + i=$(($i + 1)) +done + +make clean +git diff --compact-summary + +if test -n "$spuriosities" +then + printf '\n%s\n' "$spuriosities" + exit 87 +fi + +# vim:sw=8:ts=8:noet:nosta: