patch 9.1.0509: not possible to translate Vim script messages

Problem:  not possible to translate Vim script messages
          (RestorerZ)
Solution: implement bindtextdomain() and gettext() to support Vim script
          message translations (Christ van Willegen)

fixes: #11637
closes: #12447

Signed-off-by: Christ van Willegen <cvwillegen@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>

Filelist

@@ -221,6 +221,7 @@ SRC_ALL =	\
 		src/testdir/silent.wav \
 		src/testdir/popupbounce.vim \
 		src/testdir/crash/* \
+		src/testdir/ru_RU/LC_MESSAGES/__PACKAGE__.mo \
 		src/proto.h \
 		src/protodef.h \
 		src/proto/alloc.pro \

runtime/doc/builtin.txt

@@ -1,4 +1,4 @@
-*builtin.txt*	For Vim version 9.1.  Last change: 2024 Jun 19
+*builtin.txt*	For Vim version 9.1.  Last change: 2024 Jun 20
 
 
 		  VIM REFERENCE MANUAL	  by Bram Moolenaar
@@ -67,6 +67,8 @@ autocmd_get([{opts}])		List	return a list of autocmds
 balloon_gettext()		String	current text in the balloon
 balloon_show({expr})		none	show {expr} inside the balloon
 balloon_split({msg})		List	split {msg} as used for a balloon
+bindtextdomain({package}, {path})
+				none	bind text domain to specied path
 blob2list({blob})		List	convert {blob} into a list of numbers
 browse({save}, {title}, {initdir}, {default})
 				String	put up a file requester
@@ -277,7 +279,8 @@ gettabvar({nr}, {varname} [, {def}])
 gettabwinvar({tabnr}, {winnr}, {name} [, {def}])
 				any	{name} in {winnr} in tab page {tabnr}
 gettagstack([{nr}])		Dict	get the tag stack of window {nr}
-gettext({text})			String	lookup translation of {text}
+gettext({text} [, {package}])
+				String	lookup translation of {text}
 getwininfo([{winid}])		List	list of info about each window
 getwinpos([{timeout}])		List	X and Y coord in pixels of Vim window
 getwinposx()			Number	X coord in pixels of the Vim window
@@ -1218,6 +1221,13 @@ balloon_split({msg})					*balloon_split()*
 
 		Return type: list<any> or list<string>
 
+bindtextdomain({package}, {path})			*bindtextdomain()*
+		Bind a specific {package} to a {path} so that the
+		|gettext()| function can be used to get language-specific
+		translations for a package.  {path} is the directory name
+		for the translations. See |package-create|.
+
+		Return type: none
 
 blob2list({blob})					*blob2list()*
 		Return a List containing the number value of each byte in Blob
@@ -4978,7 +4988,7 @@ gettagstack([{winnr}])					*gettagstack()*
 		Return type: dict<any>
 
 
-gettext({text})						*gettext()*
+gettext({text} [, {package}])				*gettext()*
 		Translate String {text} if possible.
 		This is mainly for use in the distributed Vim scripts.  When
 		generating message translations the {text} is extracted by
@@ -4988,6 +4998,9 @@ gettext({text})						*gettext()*
 		For {text} double quoted strings are preferred, because
 		xgettext does not understand escaping in single quoted
 		strings.
+		When the {package} is specified, the translation is looked up
+		for that specific package. You need to specify the path to
+		look for translations with the |bindtextdomain()| function.
 
 		Return type: |String|
 

runtime/doc/repeat.txt

@@ -1,4 +1,4 @@
-*repeat.txt*    For Vim version 9.1.  Last change: 2023 May 26
+*repeat.txt*    For Vim version 9.1.  Last change: 2024 Jun 20
 
 
 		  VIM REFERENCE MANUAL    by Bram Moolenaar
@@ -735,6 +735,10 @@ Your directory layout would be like this:
    start/foobar/autoload/foo.vim	" loaded when foo command used
    start/foobar/doc/foo.txt		" help for foo.vim
    start/foobar/doc/tags		" help tags
+   start/foobar/lang/<lang_id>/LC_MESSAGES/foo.po
+					" messages for the plugin in the
+					" <lang_id> language.  These files are
+					" optional.
    opt/fooextra/plugin/extra.vim	" optional plugin, defines commands
    opt/fooextra/autoload/extra.vim	" loaded when extra command used
    opt/fooextra/doc/extra.txt		" help for extra.vim
@@ -762,6 +766,35 @@ the command after changing the plugin help: >
 	:helptags path/start/foobar/doc
 	:helptags path/opt/fooextra/doc
 
+The messages that are in the lang/<lang_id>/LC_MESSAGES/foo.po file need to be
+translated to a format that the |gettext()| function understands by running the
+msgfmt program. This will result in a lang/<lang_id>/LC_MESSAGES/foo.mo
+file. See |multilang| on how to specify languages.
+
+In your plugin, you need to call the |bindtextdomain()| function as follows.
+This assumes that the directory structure is as above: >
+	:call bindtextdomain("foo", fnamemodify(expand("<script>"), ':p:h')
+	 .. '/../lang/')
+<
+You only need to do this once. After this call, you can use: >
+	:echo gettext("Hello", "foo")
+<
+to get the text "Hello" translated to the user's preferred language (if the
+plugin messages have been translated to this language).
+
+To create the foo.po file, you need to create a foo.pot file first. The
+entries in this file need to be translated to the language(s) you want to be
+supported by your plugin.
+
+To create the foo.pot file, run the following command: >
+	cd ~/.vim/pack/start/foobar
+	make -f ~/src/vim/src/po/Makefile PACKAGE=foo \
+	PO_BASEDIR=~/src/vim/src/po PO_INPUTLIST= \
+	PO_VIM_JSLIST="plugin__foo.js plugin__bar.js \
+	autoload__foo.js" \
+	PO_VIM_INPUTLIST="plugin/foo.vim plugin/bar.vim autoload/foo.vim" \
+	foo.pot
+<
 
 Dependencies between plugins ~
 							*packload-two-steps*

runtime/doc/tags

@@ -6148,6 +6148,7 @@ beval_text-variable	eval.txt	/*beval_text-variable*
 beval_winid-variable	eval.txt	/*beval_winid-variable*
 beval_winnr-variable	eval.txt	/*beval_winnr-variable*
 binary-number	eval.txt	/*binary-number*
+bindtextdomain()	builtin.txt	/*bindtextdomain()*
 bitwise-function	usr_41.txt	/*bitwise-function*
 bitwise-shift	eval.txt	/*bitwise-shift*
 blob	eval.txt	/*blob*
@@ -10598,6 +10599,7 @@ termdebug-mappings	terminal.txt	/*termdebug-mappings*
 termdebug-prompt	terminal.txt	/*termdebug-prompt*
 termdebug-starting	terminal.txt	/*termdebug-starting*
 termdebug-stepping	terminal.txt	/*termdebug-stepping*
+termdebug-timeout	terminal.txt	/*termdebug-timeout*
 termdebug-variables	terminal.txt	/*termdebug-variables*
 termdebug_disasm_window	terminal.txt	/*termdebug_disasm_window*
 termdebug_map_K	terminal.txt	/*termdebug_map_K*

runtime/doc/usr_41.txt

@@ -798,6 +798,7 @@ String manipulation:					*string-functions*
 	execute()		execute an Ex command and get the output
 	win_execute()		like execute() but in a specified window
 	trim()			trim characters from a string
+	bindtextdomain()	set message lookup translation base path
 	gettext()		lookup message translation
 
 List manipulation:					*list-functions*

runtime/doc/version9.txt

@@ -41563,6 +41563,9 @@ Support for the XDG Desktop Specification |xdg-base-dir|
 Support highlighting the matched text for insert-mode completion and
 command-line completion in |ins-completion-menu|.
 
+Support for translating messages in Vim script plugins using the |gettext()|
+and |bindtextdomain()| functions.
+
 							*changed-9.2*
 Changed~
 -------
@@ -41579,6 +41582,7 @@ Various syntax, indent and other plugins were added.
 
 Functions: ~
 
+|bindtextdomain()|	set message lookup translation base path
 |diff()|		diff two Lists of strings
 |filecopy()|		copy a file {from} to {to}
 |foreach()|		apply function to List items

src/auto/configure

@@ -15876,6 +15876,30 @@ then :
 
 fi
 
+      { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for dgettext" >&5
+printf %s "checking for dgettext... " >&6; }
+      cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <libintl.h>
+int
+main (void)
+{
+dgettext("Test", "Test");
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+printf "%s\n" "yes" >&6; }; printf "%s\n" "#define HAVE_DGETTEXT 1" >>confdefs.h
+
+else $as_nop
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
             { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _nl_msg_cat_cntr" >&5
 printf %s "checking for _nl_msg_cat_cntr... " >&6; }
       cat confdefs.h - <<_ACEOF >conftest.$ac_ext

src/config.h.in

@@ -222,7 +222,6 @@
 #undef HAVE_UNSETENV
 #undef HAVE_USLEEP
 #undef HAVE_UTIME
-#undef HAVE_BIND_TEXTDOMAIN_CODESET
 #undef HAVE_MBLEN
 #undef HAVE_TIMER_CREATE
 #undef HAVE_CLOCK_GETTIME
@@ -424,6 +423,12 @@
 /* Define if there is a working gettext(). */
 #undef HAVE_GETTEXT
 
+/* Define if there is a working bind_textdomain_codeset(). */
+#undef HAVE_BIND_TEXTDOMAIN_CODESET
+
+/* Define if there is a working dgettext(). */
+#undef HAVE_DGETTEXT
+
 /* Define if _nl_msg_cat_cntr is present. */
 #undef HAVE_NL_MSG_CAT_CNTR
 

src/configure.ac

@@ -4497,6 +4497,12 @@ if test "$enable_nls" = "yes"; then
       AC_SUBST(MAKEMO)
       dnl this was added in GNU gettext 0.10.36
       AC_CHECK_FUNCS(bind_textdomain_codeset)
+      AC_MSG_CHECKING([for dgettext])
+      AC_LINK_IFELSE([AC_LANG_PROGRAM(
+		[#include <libintl.h>],
+		[dgettext("Test", "Test");])],
+		AC_MSG_RESULT([yes]); AC_DEFINE(HAVE_DGETTEXT),
+		AC_MSG_RESULT([no]))
       dnl _nl_msg_cat_cntr is required for GNU gettext
       AC_MSG_CHECKING([for _nl_msg_cat_cntr])
       AC_LINK_IFELSE([AC_LANG_PROGRAM(

src/evalfunc.c

@@ -28,6 +28,7 @@ static void f_balloon_show(typval_T *argvars, typval_T *rettv);
 static void f_balloon_split(typval_T *argvars, typval_T *rettv);
 # endif
 #endif
+static void f_bindtextdomain(typval_T *argvars, typval_T *rettv);
 static void f_byte2line(typval_T *argvars, typval_T *rettv);
 static void f_call(typval_T *argvars, typval_T *rettv);
 static void f_changenr(typval_T *argvars, typval_T *rettv);
@@ -1824,6 +1825,8 @@ static funcentry_T global_functions[] =
 	    NULL
 #endif
 			},
+    {"bindtextdomain",	2, 2, 0,	    arg2_string,
+			ret_void,	    f_bindtextdomain},
     {"blob2list",	1, 1, FEARG_1,	    arg1_blob,
 			ret_list_number,    f_blob2list},
     {"browse",		4, 4, 0,	    arg4_browse,
@@ -2154,7 +2157,7 @@ static funcentry_T global_functions[] =
 			ret_any,	    f_gettabwinvar},
     {"gettagstack",	0, 1, FEARG_1,	    arg1_number,
 			ret_dict_any,	    f_gettagstack},
-    {"gettext",		1, 1, FEARG_1,	    arg1_string,
+    {"gettext",		1, 2, FEARG_1,	    arg2_string,
 			ret_string,	    f_gettext},
     {"getwininfo",	0, 1, FEARG_1,	    arg1_number,
 			ret_list_dict_any,  f_getwininfo},
@@ -3476,6 +3479,24 @@ get_buf_arg(typval_T *arg)
     return buf;
 }
 
+/*
+ * "bindtextdomain(package, path)" function
+ */
+    static void
+f_bindtextdomain(typval_T *argvars UNUSED, typval_T *rettv UNUSED)
+{
+    if (check_for_nonempty_string_arg(argvars, 0) == FAIL
+	    || check_for_nonempty_string_arg(argvars, 1) == FAIL)
+	return;
+
+    if (strcmp((const char *)argvars[0].vval.v_string, VIMPACKAGE) == 0)
+	semsg(_(e_invalid_argument_str), tv_get_string(&argvars[0]));
+    else
+	bindtextdomain((const char *)argvars[0].vval.v_string, (const char *)argvars[1].vval.v_string);
+
+    return;
+}
+
 /*
  * "byte2line(byte)" function
  */
@@ -6033,11 +6054,39 @@ f_gettagstack(typval_T *argvars, typval_T *rettv)
     static void
 f_gettext(typval_T *argvars, typval_T *rettv)
 {
-    if (check_for_nonempty_string_arg(argvars, 0) == FAIL)
+#if defined(HAVE_BIND_TEXTDOMAIN_CODESET)
+    char *prev = NULL;
+#endif
+
+    if (check_for_nonempty_string_arg(argvars, 0) == FAIL
+	|| check_for_opt_string_arg(argvars, 1) == FAIL)
 	return;
 
     rettv->v_type = VAR_STRING;
-    rettv->vval.v_string = vim_strsave((char_u *)_(argvars[0].vval.v_string));
+
+    if (argvars[1].v_type == VAR_STRING &&
+	    argvars[1].vval.v_string != NULL &&
+	    *(argvars[1].vval.v_string) != NUL)
+    {
+#if defined(HAVE_BIND_TEXTDOMAIN_CODESET)
+	prev = bind_textdomain_codeset((const char *)argvars[1].vval.v_string, (char *)p_enc);
+#endif
+
+#if defined(HAVE_DGETTEXT)
+	rettv->vval.v_string = vim_strsave((char_u *)dgettext((const char *)argvars[1].vval.v_string, (const char *)argvars[0].vval.v_string));
+#else
+	textdomain((const char *)argvars[1].vval.v_string);
+	rettv->vval.v_string = vim_strsave((char_u *)_(argvars[0].vval.v_string));
+	textdomain(VIMPACKAGE);
+#endif
+
+#if defined(HAVE_BIND_TEXTDOMAIN_CODESET)
+	if (prev != NULL)
+	    bind_textdomain_codeset((const char *)argvars[1].vval.v_string, prev);
+#endif
+    }
+    else
+	rettv->vval.v_string = vim_strsave((char_u *)_(argvars[0].vval.v_string));
 }
 
 // for VIM_VERSION_ defines

src/po/Make_all.mak

@@ -189,8 +189,8 @@ PO_VIM_INPUTLIST = \
 	../../runtime/defaults.vim
 
 PO_VIM_JSLIST = \
-	optwin.js \
+	________runtime__optwin.js \
-	defaults.js
+	________runtime__defaults.js
 
 # Arguments for xgettext to pick up messages to translate from the source code.
 XGETTEXT_KEYWORDS = --keyword=_ --keyword=N_ --keyword=NGETTEXT:1,2 --keyword=PLURAL_MSG:2,4

src/po/Makefile

@@ -1,17 +1,18 @@
 # Makefile for the Vim message translations.
+PO_BASEDIR = .
 
 # Include stuff found by configure.
-include ../auto/config.mk
+include $(PO_BASEDIR)/../auto/config.mk
 
 # Get LANGUAGES, MOFILES, MOCONVERTED and others.
-include Make_all.mak
+include $(PO_BASEDIR)/Make_all.mak
 
 # Note: ja.sjis, *.cp1250 and zh_CN.cp936 are only for MS-Windows, they are
 # not installed on Unix.
 
 PACKAGE = vim
 SHELL = /bin/sh
-VIM = ../vim
+VIM = $(PO_BASEDIR)/../vim
 
 # MacOS sed is locale aware, set $LANG to avoid problems.
 SED = LANG=C sed
@@ -261,13 +262,13 @@ PO_INPUTLIST = \
 
 $(PACKAGE).pot: $(PO_INPUTLIST) $(PO_VIM_INPUTLIST)
 	# Convert the Vim scripts to (what looks like) Javascript.
-	$(VIM) -u NONE --not-a-term -S tojavascript.vim $(PACKAGE).pot $(PO_VIM_INPUTLIST)
+	$(VIM) -u NONE --not-a-term -S $(PO_BASEDIR)/tojavascript.vim $(PACKAGE).pot $(PO_VIM_INPUTLIST)
 	# Create vim.pot.
 	$(XGETTEXT) --default-domain=$(PACKAGE) --add-comments \
 		$(XGETTEXT_KEYWORDS) $(PO_INPUTLIST) $(PO_VIM_JSLIST)
 	mv -f $(PACKAGE).po $(PACKAGE).pot
 	# Fix Vim scripts names, so that "gf" works.
-	$(VIM) -u NONE --not-a-term -S fixfilenames.vim $(PACKAGE).pot $(PO_VIM_INPUTLIST)
+	$(VIM) -u NONE --not-a-term -S $(PO_BASEDIR)/fixfilenames.vim $(PACKAGE).pot $(PO_VIM_INPUTLIST)
 	# Delete the temporary files.
 	rm *.js
 

src/po/fixfilenames.vim

@@ -4,7 +4,7 @@
 set shortmess+=A
 
 for name in argv()[1:]
-  let jsname = fnamemodify(name, ":t:r") .. ".js"
+  let jsname = fnamemodify(name, ":r:gs?\\~?_?:gs?\\.?_?:gs?/?__?:gs?\\?__?") .. ".js"
   exe "%s+" .. jsname .. "+" .. substitute(name, '\\', '/', 'g') .. "+"
 endfor
 

src/po/tojavascript.vim

@@ -13,7 +13,7 @@ for name in argv()[1:]
   g/^\s*set .*"/s/.*//
 
   " Write as .js file, xgettext recognizes them
-  exe 'w! ' .. fnamemodify(name, ":t:r") .. ".js"
+  exe 'w! ' .. fnamemodify(name, ":r:gs?\\~?_?:gs?\\.?_?:gs?/?__?:gs?\\?__?") .. ".js"
 endfor
 
 quit

src/testdir/Make_all.mak

@@ -161,6 +161,9 @@ NEW_TESTS = \
 	test_function_lists \
 	test_ga \
 	test_getcwd \
+	test_gettext \
+	test_gettext_cp1251 \
+	test_gettext_utf8 \
 	test_getvar \
 	test_gf \
 	test_glob2regpat \
@@ -420,6 +423,9 @@ NEW_TESTS_RES = \
 	test_functions.res \
 	test_function_lists.res \
 	test_getcwd.res \
+	test_gettext.res \
+	test_gettext_cp1251.res \
+	test_gettext_utf8.res \
 	test_getvar.res \
 	test_gf.res \
 	test_gn.res \

src/testdir/test_functions.vim

@@ -3865,11 +3865,6 @@ func Test_default_arg_value()
   call assert_equal('msg', HasDefault())
 endfunc
 
-" Test for gettext()
-func Test_gettext()
-  call assert_fails('call gettext(1)', 'E1174:')
-endfunc
-
 func Test_builtin_check()
   call assert_fails('let g:["trim"] = {x -> " " .. x}', 'E704:')
   call assert_fails('let g:.trim = {x -> " " .. x}', 'E704:')

src/testdir/test_gettext.vim

@@ -0,0 +1,16 @@
+source check.vim
+
+" Test for gettext()
+func Test_gettext()
+  call assert_fails('call bindtextdomain("test")', 'E119:')
+  call assert_fails('call bindtextdomain("vim", "test")', 'E475:')
+
+  call assert_fails('call gettext(1)', 'E1174:')
+  call assert_equal('xxxTESTxxx', gettext("xxxTESTxxx"))
+
+  call assert_equal('xxxTESTxxx', gettext("xxxTESTxxx", "vim"))
+  call assert_equal('xxxTESTxxx', gettext("xxxTESTxxx", "__PACKAGE__"))
+  call assert_equal('ERROR: ', gettext("ERROR: ", "__PACKAGE__"))
+endfunc
+
+" vim: shiftwidth=2 sts=2 expandtab

src/testdir/test_gettext_cp1251.vim

@@ -0,0 +1,22 @@
+source check.vim
+
+" Test for gettext()
+func Test_gettext()
+  set encoding=cp1251
+  call bindtextdomain("__PACKAGE__", getcwd())
+  try
+    language ru_RU
+    call assert_equal('<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>: ', gettext("ERROR: ", "__PACKAGE__"))
+  catch /^Vim\%((\a\+)\)\=:E197:/
+    throw "Skipped: not possible to set locale to ru (missing?)"
+  endtry
+  try
+    language en_GB.UTF-8
+    call assert_equal('ERROR: ', gettext("ERROR: ", "__PACKAGE__"))
+  catch /^Vim\%((\a\+)\)\=:E197:/
+    throw "Skipped: not possible to set locale to en (missing?)"
+  endtry
+  set encoding&
+endfunc
+
+" vim: shiftwidth=2 sts=2 expandtab

src/testdir/test_gettext_utf8.vim

@@ -0,0 +1,22 @@
+source check.vim
+
+" Test for gettext()
+func Test_gettext()
+  set encoding=utf-8
+  call bindtextdomain("__PACKAGE__", getcwd())
+  try
+    language ru_RU
+    call assert_equal('ОШИБКА: ', gettext("ERROR: ", "__PACKAGE__"))
+  catch /^Vim\%((\a\+)\)\=:E197:/
+    throw "Skipped: not possible to set locale to ru (missing?)"
+  endtry
+  try
+    language en_GB.UTF-8
+    call assert_equal('ERROR: ', gettext("ERROR: ", "__PACKAGE__"))
+  catch /^Vim\%((\a\+)\)\=:E197:/
+    throw "Skipped: not possible to set locale to en (missing?)"
+  endtry
+  set encoding&
+endfunc
+
+" vim: shiftwidth=2 sts=2 expandtab

src/version.c

@@ -704,6 +704,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    509,
 /**/
     508,
 /**/