mirror of
https://github.com/vim/vim
synced 2025-07-16 09:12:00 +00:00
patch 9.1.0681: tests: Analyzing failed screendumps is hard
Problem: tests: Analyzing failed screendumps is hard
Solution: Facilitate the viewing of rendered screendumps under src/
add some documentation on how to use the viewdumps.vim
script (Aliaksei Budavei)
With the submitted "viewdumps.vim" script, a few manual
steps in typical workflows (see below) can be automated.
The updated "README.txt" contains additional information.
============================================================
Reviewing LOCAL failed screendump tests can be arranged as
follows:
1) Run tests and generate screendumps:
------------------------------------------------------------
cd /path/to/fork/src/testdir
make
------------------------------------------------------------
2) Examine the screendumps from the "failed" directory:
------------------------------------------------------------
../vim -u NONE -S viewdumps.vim
------------------------------------------------------------
============================================================
Reviewing UPLOADED failed screendump tests can be arranged
as follows (it can be further locally scripted):
1) Fetch an artifact with failed screendumps from
"github.com/vim/vim/actions/runs/A_ID/artifacts/B_ID".
2) Extract the archived files:
------------------------------------------------------------
unzip /tmp/failed-tests.zip -d /tmp
------------------------------------------------------------
3) Set up the "dumps" directory. Create a symlink to
"/path/to/fork/dirs/dumps" in the extracted directories so
that term_dumpdiff() can be used. (The lookup algorithm
resolves "dumps" for every loaded filename. So, with
"/tmp/src/testdir/failed/*.dump" files passed as script
arguments, the algorithm will make the files in
"/tmp/src/testdir/dumps" queried.)
------------------------------------------------------------
cd /path/to/fork
ln -s $(pwd)/src/testdir/dumps /tmp/src/testdir/dumps
------------------------------------------------------------
4) Examine the extracted screendumps:
------------------------------------------------------------
./src/vim -u NONE -S src/testdir/viewdumps.vim \
/tmp/src/testdir/failed/*.dump
------------------------------------------------------------
5) Clean up:
------------------------------------------------------------
unlink /tmp/src/testdir/dumps
rm -rf /tmp/src
------------------------------------------------------------
============================================================
Reviewing SUBMITTED FOR PULL REQUEST screendump tests can be
arranged as follows (it can be further locally scripted):
1) List the fetched changeset and write the changed "dumps"
filenames to "/tmp/filelist":
------------------------------------------------------------
cd /path/to/fork
git switch prs/1234
git diff-index --relative=src/testdir/dumps/ \
--name-only prs/1234~1 > /tmp/filelist
------------------------------------------------------------
2) Reconcile relative filepaths, and copy next-to-be-updated
"dumps" files in the "failed" directory (note the missing
new screendumps, if any):
------------------------------------------------------------
git switch master
cd src/testdir/dumps
test -d ../failed || mkdir ../failed
cp -t ../failed $(cat /tmp/filelist)
------------------------------------------------------------
3) Remember about the introduced INVERTED relation between
"dumps" and "failed", i.e. the files to be committed are in
"dumps" already and their previous versions are in "failed";
therefore, copy the missing new screendumps from "dumps" to
"failed" (otherwise these won't be shown):
------------------------------------------------------------
git switch prs/1234
cp -t ../failed foo_10.dump foo_11.dump foo_12.dump
------------------------------------------------------------
4) Examine the screendumps from the "failed" directory (new
screendumps will be shown with no difference between their
versions):
------------------------------------------------------------
cd ..
../vim -u NONE -S viewdumps.vim
------------------------------------------------------------
closes: #15515
Signed-off-by: Aliaksei Budavei <0x000c70@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>
This commit is contained in:
Aliaksei Budavei
committed by
Christian Brabandt
Christian Brabandt
parent
175a87c7f1
commit
6bff6a2fa4
2
Filelist
2
Filelist
@ -185,6 +185,7 @@ SRC_ALL = \
|
||||
src/testdir/sautest/autoload/*.vim \
|
||||
src/testdir/testluaplugin/lua/testluaplugin/*.lua \
|
||||
src/testdir/check.vim \
|
||||
src/testdir/commondumps.vim \
|
||||
src/testdir/gui_init.vim \
|
||||
src/testdir/gui_preinit.vim \
|
||||
src/testdir/mouse.vim \
|
||||
@ -198,6 +199,7 @@ SRC_ALL = \
|
||||
src/testdir/summarize.vim \
|
||||
src/testdir/term_util.vim \
|
||||
src/testdir/view_util.vim \
|
||||
src/testdir/viewdumps.vim \
|
||||
src/testdir/test[0-9]*.ok \
|
||||
src/testdir/test77a.ok \
|
||||
src/testdir/test77a.com \
|
||||
|
||||
@ -124,8 +124,8 @@ verify that the tests fail. Then you know your changes are covered by the
|
||||
test.
|
||||
|
||||
|
||||
Viewing generated screendumps
|
||||
-----------------------------
|
||||
Viewing generated screendumps (local)
|
||||
-------------------------------------
|
||||
|
||||
You may also wish to look at the whole batch of failed screendumps after
|
||||
running "make test". Source the "viewdumps.vim" script for this task:
|
||||
@ -151,5 +151,60 @@ At any time, you can add, list, and abandon other screendumps:
|
||||
The listing of argument commands can be found under :help buffer-list.
|
||||
|
||||
|
||||
Viewing generated screendumps (from a CI-uploaded artifact)
|
||||
-----------------------------------------------------------
|
||||
|
||||
After you have downloaded an artifact archive containing failed screendumps
|
||||
and extracted its files in a temporary directory, you need to set up a "dumps"
|
||||
directory by creating a symlink:
|
||||
|
||||
cd /path/to/fork
|
||||
ln -s $(pwd)/runtime/syntax/testdir/dumps \
|
||||
/tmp/runtime/syntax/testdir/dumps
|
||||
|
||||
You can now examine the extracted screendumps:
|
||||
|
||||
./src/vim --clean -S runtime/syntax/testdir/viewdumps.vim \
|
||||
/tmp/runtime/syntax/testdir/failed/*.dump
|
||||
|
||||
|
||||
Viewing generated screendumps (submitted for a pull request)
|
||||
------------------------------------------------------------
|
||||
|
||||
First, you need to check out the topic branch with the proposed changes and
|
||||
write down a difference list between the HEAD commit (index) and its parent
|
||||
commit with respect to the changed "dumps" filenames:
|
||||
|
||||
cd /path/to/fork
|
||||
git switch prs/1234
|
||||
git diff-index --relative=runtime/syntax/testdir/dumps/ \
|
||||
--name-only prs/1234~1 > /tmp/filelist
|
||||
|
||||
Then, you need to check out the master branch, change the current working
|
||||
directory to reconcile relative filepaths written in the filenames list, copy
|
||||
in the "failed" directory the old "dumps" files, whose names are on the same
|
||||
list, and follow it by checking out the topic branch:
|
||||
|
||||
git switch master
|
||||
cd runtime/syntax/testdir/dumps
|
||||
cp -t ../failed $(cat /tmp/filelist)
|
||||
git switch prs/1234
|
||||
|
||||
Make note of any missing new screendumps. Please remember about the
|
||||
introduced INVERTED relation between "dumps" and "failed", i.e. the files to
|
||||
be committed are in "dumps" already and their old versions are in "failed".
|
||||
Therefore, you need to copy the missing new screendumps from "dumps" to
|
||||
"failed":
|
||||
|
||||
cp -t ../failed foo_10.dump foo_11.dump foo_12.dump
|
||||
|
||||
After you have changed the current working directory to its parent directory,
|
||||
you can now examine the screendumps from the "failed" directory (note that new
|
||||
screendumps will be shown with no difference between their versions):
|
||||
|
||||
cd ..
|
||||
../../../src/vim --clean -S viewdumps.vim
|
||||
|
||||
|
||||
TODO: run test for one specific filetype
|
||||
TODO: test syncing by jumping around
|
||||
|
||||
@ -1,44 +1,9 @@
|
||||
vim9script
|
||||
|
||||
# Support sourcing this script from this directory or any other directory in
|
||||
# the direct path that leads to the project's root directory.
|
||||
const failed_path: string = finddir('failed', getcwd() .. '/**', -1)
|
||||
->filter(((cwdpath: string) => (_: number, dirpath: string) =>
|
||||
cwdpath =~ '\<syntax\>' || dirpath =~ '\<syntax\>')(getcwd()))
|
||||
->get(-1, '') .. '/'
|
||||
var error: string = null_string
|
||||
|
||||
if failed_path == '/'
|
||||
error = 'No such directory: "failed"'
|
||||
else
|
||||
const failed_fnames: string = failed_path .. readdir(failed_path,
|
||||
(fname: string) => fname =~ '^.\+\.dump$')
|
||||
->join(' ' .. failed_path)
|
||||
|
||||
if failed_fnames =~ 'failed/$'
|
||||
error = 'No such file: "*.dump"'
|
||||
else
|
||||
exec ':0argedit ' .. failed_fnames
|
||||
buffers
|
||||
endif
|
||||
endif
|
||||
|
||||
def Render()
|
||||
const failed_fname: string = bufname()
|
||||
try
|
||||
setlocal suffixesadd=.dump
|
||||
const dumps_fname: string = findfile(
|
||||
fnamemodify(failed_fname, ':p:t'),
|
||||
fnamemodify(failed_fname, ':p:h:h') .. '/dumps')
|
||||
if filereadable(dumps_fname)
|
||||
term_dumpdiff(failed_fname, dumps_fname)
|
||||
else
|
||||
term_dumpload(failed_fname)
|
||||
endif
|
||||
finally
|
||||
exec 'bwipeout ' .. failed_fname
|
||||
endtry
|
||||
enddef
|
||||
exec 'source ' .. (((cwdpath: string) => cwdpath
|
||||
->strpart(0, cwdpath->strridx('/vim')))(getcwd()))
|
||||
.. '/vim/src/testdir/commondumps.vim'
|
||||
g:Init('\<syntax\>', -1)
|
||||
|
||||
# THE FOLLOWING SETTINGS PERTAIN TO "input/" FILES THAT ARE LIKELY TO BE
|
||||
# LOADED SIDE BY SIDE WHENEVER BATCHES OF NEW SCREENDUMPS ARE GENERATED.
|
||||
@ -56,22 +21,4 @@ set display=lastline ruler scrolloff=5 t_ZH= t_ZR=
|
||||
# Anticipate non-Latin-1 characters in "input/" files.
|
||||
set encoding=utf-8 termencoding=utf-8
|
||||
|
||||
autocmd_add([{
|
||||
replace: true,
|
||||
group: 'viewdumps',
|
||||
event: 'BufRead',
|
||||
pattern: '*.dump',
|
||||
cmd: 'Render()',
|
||||
}])
|
||||
|
||||
# Unconditionally help, in case a list of filenames is passed to the command,
|
||||
# the first terminal window with its BufRead event.
|
||||
silent doautocmd viewdumps BufRead
|
||||
|
||||
if error != null_string
|
||||
# Instead of sleeping, fill half a window with blanks and prompt hit-enter.
|
||||
echom error .. repeat("\x20", (winwidth(0) * (winheight(0) / 2) - strlen(error)))
|
||||
error = null_string
|
||||
endif
|
||||
|
||||
# vim:fdm=syntax:sw=2:ts=8:noet:nolist:nosta:
|
||||
|
||||
@ -97,8 +97,7 @@ tests are successful, then this file will be an empty file.
|
||||
|
||||
- To execute only specific test functions, add a second argument:
|
||||
|
||||
$ ../vim -u NONE -S runtest.vim test_channel.vim open_delay
|
||||
|
||||
$ ../vim -u NONE -S runtest.vim test_channel.vim open_delay
|
||||
|
||||
- To run all the tests:
|
||||
|
||||
@ -120,9 +119,82 @@ tests are successful, then this file will be an empty file.
|
||||
|
||||
$ make clean
|
||||
|
||||
# ANALYZE FAILED SCREENDUMPS FROM CI:
|
||||
|
||||
See the file ../../runtime/syntax/testdir/README.txt section
|
||||
"Viewing generated screendumps" on how to analyze failed screen dumps
|
||||
(from CI or locally) using the provided Vim script
|
||||
../../runtime/syntax/testdir/viewdumps.vim in a more automatic way.
|
||||
VIEWING GENERATED SCREENDUMPS (local):
|
||||
|
||||
You may also wish to look at the whole batch of failed screendumps after
|
||||
running "make". Source the "viewdumps.vim" script for this task:
|
||||
|
||||
$ ../vim -u NONE -S viewdumps.vim \
|
||||
[dumps/Test_xxd_*.dump ...]
|
||||
|
||||
By default, all screendumps found in the "failed" directory will be added to
|
||||
the argument list and then the first one will be loaded. Loaded screendumps
|
||||
that bear filenames of screendumps found in the "dumps" directory will be
|
||||
rendering the contents of any such pair of files and the difference between
|
||||
them (:help term_dumpdiff()); otherwise, they will be rendering own contents
|
||||
(:help term_dumpload()). Remember to execute :edit when occasionally you see
|
||||
raw file contents instead of rendered.
|
||||
|
||||
At any time, you can add, list, and abandon other screendumps:
|
||||
|
||||
:$argedit dumps/Test_spell_*.dump
|
||||
:args
|
||||
:qall
|
||||
|
||||
The listing of argument commands can be found under :help buffer-list.
|
||||
|
||||
|
||||
VIEWING GENERATED SCREENDUMPS (from a CI-uploaded artifact):
|
||||
|
||||
After you have downloaded an artifact archive containing failed screendumps
|
||||
and extracted its files in a temporary directory, you need to set up a "dumps"
|
||||
directory by creating a symlink:
|
||||
|
||||
$ cd /path/to/fork
|
||||
$ ln -s $(pwd)/src/testdir/dumps /tmp/src/testdir/dumps
|
||||
|
||||
You can now examine the extracted screendumps:
|
||||
|
||||
$ ./src/vim -u NONE -S src/testdir/viewdumps.vim \
|
||||
/tmp/src/testdir/failed/*.dump
|
||||
|
||||
|
||||
VIEWING GENERATED SCREENDUMPS (submitted for a pull request):
|
||||
|
||||
First, you need to check out the topic branch with the proposed changes and
|
||||
write down a difference list between the HEAD commit (index) and its parent
|
||||
commit with respect to the changed "dumps" filenames:
|
||||
|
||||
$ cd /path/to/fork
|
||||
$ git switch prs/1234
|
||||
$ git diff-index --relative=src/testdir/dumps/ \
|
||||
--name-only prs/1234~1 > /tmp/filelist
|
||||
|
||||
Then, you need to check out the master branch, change the current working
|
||||
directory to reconcile relative filepaths written in the filenames list,
|
||||
possibly create an empty "failed" directory, copy in this directory the old
|
||||
"dumps" files, whose names are on the same list, and follow it by checking out
|
||||
the topic branch:
|
||||
|
||||
$ git switch master
|
||||
$ cd src/testdir/dumps
|
||||
$ test -d ../failed || mkdir ../failed
|
||||
$ cp -t ../failed $(cat /tmp/filelist)
|
||||
$ git switch prs/1234
|
||||
|
||||
Make note of any missing new screendumps. Please remember about the
|
||||
introduced INVERTED relation between "dumps" and "failed", i.e. the files to
|
||||
be committed are in "dumps" already and their old versions are in "failed".
|
||||
Therefore, you need to copy the missing new screendumps from "dumps" to
|
||||
"failed":
|
||||
|
||||
$ cp -t ../failed foo_10.dump foo_11.dump foo_12.dump
|
||||
|
||||
After you have changed the current working directory to its parent directory,
|
||||
you can now examine the screendumps from the "failed" directory (note that new
|
||||
screendumps will be shown with no difference between their versions):
|
||||
|
||||
$ cd ..
|
||||
$ ../vim -u NONE -S viewdumps.vim
|
||||
|
||||
|
||||
76
src/testdir/commondumps.vim
Normal file
76
src/testdir/commondumps.vim
Normal file
@ -0,0 +1,76 @@
|
||||
vim9script
|
||||
|
||||
# (Script-local.)
|
||||
#
|
||||
# Render a loaded screendump file or the difference of a loaded screendump
|
||||
# file and its namesake file from the "dumps" directory.
|
||||
def Render()
|
||||
const failed_fname: string = bufname()
|
||||
try
|
||||
setlocal suffixesadd=.dump
|
||||
const dumps_fname: string = findfile(
|
||||
fnamemodify(failed_fname, ':p:t'),
|
||||
fnamemodify(failed_fname, ':p:h:h') .. '/dumps')
|
||||
if filereadable(dumps_fname)
|
||||
term_dumpdiff(failed_fname, dumps_fname)
|
||||
else
|
||||
term_dumpload(failed_fname)
|
||||
endif
|
||||
finally
|
||||
exec 'bwipeout ' .. failed_fname
|
||||
endtry
|
||||
enddef
|
||||
|
||||
# Search for the "failed" directory in the passed _subtreedirname_ directories
|
||||
# (usually "\<src\>" or "\<syntax\>") and, if found, select its passed _count_
|
||||
# occurence, add all its "*.dump" files to the argument list and list them;
|
||||
# also define a BufRead autocommand that would invoke "Render()" for every
|
||||
# "*.dump" file.
|
||||
def g:Init(subtreedirname: string, count: number)
|
||||
# Support sourcing this script from any directory in the direct path that
|
||||
# leads to the project's root directory.
|
||||
const failed_path: string = finddir('failed', getcwd() .. '/**', -1)
|
||||
->filter(((cwdpath: string, parentdirname: string) =>
|
||||
(_: number, dirpath: string) =>
|
||||
cwdpath =~ parentdirname || dirpath =~ parentdirname)(
|
||||
getcwd(),
|
||||
subtreedirname))
|
||||
->get(count, '') .. '/'
|
||||
var error: string = null_string
|
||||
|
||||
if failed_path == '/'
|
||||
error = 'No such directory: "failed"'
|
||||
else
|
||||
const failed_fnames: string = failed_path .. readdir(failed_path,
|
||||
(fname: string) => fname =~ '^.\+\.dump$')
|
||||
->join(' ' .. failed_path)
|
||||
|
||||
if failed_fnames =~ 'failed/$'
|
||||
error = 'No such file: "*.dump"'
|
||||
else
|
||||
exec ':0argedit ' .. failed_fnames
|
||||
buffers
|
||||
endif
|
||||
endif
|
||||
|
||||
autocmd_add([{
|
||||
replace: true,
|
||||
group: 'viewdumps',
|
||||
event: 'BufRead',
|
||||
pattern: '*.dump',
|
||||
cmd: 'Render()',
|
||||
}])
|
||||
|
||||
# Unconditionally help, in case a list of filenames is passed to the
|
||||
# command, the first terminal window with its BufRead event.
|
||||
silent doautocmd viewdumps BufRead
|
||||
|
||||
if error != null_string
|
||||
# Instead of sleeping, fill half a window with blanks and prompt
|
||||
# hit-enter.
|
||||
echom error .. repeat("\x20",
|
||||
(winwidth(0) * (winheight(0) / 2) - strlen(error)))
|
||||
endif
|
||||
enddef
|
||||
|
||||
# vim:fdm=syntax:sw=2:ts=8:noet:nolist:nosta:
|
||||
11
src/testdir/viewdumps.vim
Normal file
11
src/testdir/viewdumps.vim
Normal file
@ -0,0 +1,11 @@
|
||||
vim9script
|
||||
|
||||
exec 'source ' .. (((cwdpath: string) => cwdpath
|
||||
->strpart(0, cwdpath->strridx('/vim')))(getcwd()))
|
||||
.. '/vim/src/testdir/commondumps.vim'
|
||||
g:Init('\<src\>', 0)
|
||||
|
||||
# Match ":language" of runtest.vim.
|
||||
language messages C
|
||||
|
||||
# vim:fdm=syntax:sw=2:ts=8:noet:nolist:nosta:
|
||||
@ -704,6 +704,8 @@ static char *(features[]) =
|
||||
|
||||
static int included_patches[] =
|
||||
{ /* Add new patch number below this line */
|
||||
/**/
|
||||
681,
|
||||
/**/
|
||||
680,
|
||||
/**/
|
||||
|
||||
Reference in New Issue
Block a user