patch 8.2.2912: MS-Windows: most users expect using Unicode

Problem:    MS-Windows: most users expect using Unicode.
Solution:   Default 'encoding' to utf-8 on MS-Windows. (Ken Takata,
            closes #3907)

runtime/doc/options.txt

@@ -2828,7 +2828,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 	|setcellwidths()| function to change the behavior.
 
 					*'encoding'* *'enc'* *E543*
-'encoding' 'enc'	string (default: "latin1" or value from $LANG)
+'encoding' 'enc'	string (default for MS-Windows: "utf-8",
+				otherwise: value from $LANG or "latin1")
 			global
 	Sets the character encoding used inside Vim.  It applies to text in
 	the buffers, registers, Strings in expressions, text stored in the

runtime/doc/starting.txt

@@ -1036,7 +1036,7 @@ giving the mapping.
 
 
 Defaults without a .vimrc file ~
-							*defaults.vim*
+							*defaults.vim* *E1187*
 If Vim is started normally and no user vimrc file is found, the
 $VIMRUNTIME/defaults.vim script is loaded.  This will set 'compatible' off,
 switch on syntax highlighting and a few more things.  See the script for

src/errors.h

@@ -413,3 +413,7 @@ EXTERN char e_missing_redir_end[]
 	INIT(= N_("E1185: Missing :redir END"));
 EXTERN char e_expression_does_not_result_in_value_str[]
 	INIT(= N_("E1186: Expression does not result in a value: %s"));
+EXTERN char e_failed_to_source_defaults[]
+	INIT(= N_("E1187: Failed to source defaults.vim"));
+EXTERN char e_cannot_open_terminal_from_command_line_window[]
+	INIT(= N_("E1188: Cannot open a terminal from the command line window"));

src/main.c

@@ -3128,7 +3128,11 @@ source_startup_scripts(mparm_T *parmp)
     if (parmp->use_vimrc != NULL)
     {
 	if (STRCMP(parmp->use_vimrc, "DEFAULTS") == 0)
-	    do_source((char_u *)VIM_DEFAULTS_FILE, FALSE, DOSO_NONE, NULL);
+	{
+	    if (do_source((char_u *)VIM_DEFAULTS_FILE, FALSE, DOSO_NONE, NULL)
+									 != OK)
+		emsg(e_failed_to_source_defaults);
+	}
 	else if (STRCMP(parmp->use_vimrc, "NONE") == 0
 				     || STRCMP(parmp->use_vimrc, "NORC") == 0)
 	{
@@ -3200,7 +3204,9 @@ source_startup_scripts(mparm_T *parmp)
 		&& !has_dash_c_arg)
 	    {
 		// When no .vimrc file was found: source defaults.vim.
-		do_source((char_u *)VIM_DEFAULTS_FILE, FALSE, DOSO_NONE, NULL);
+		if (do_source((char_u *)VIM_DEFAULTS_FILE, FALSE, DOSO_NONE,
+								 NULL) == FAIL)
+		    emsg(e_failed_to_source_defaults);
 	    }
 	}
 

src/mbyte.c

@@ -4456,7 +4456,7 @@ enc_canonize(char_u *enc)
 	// Use the default encoding as it's found by set_init_1().
 	r = get_encoding_default();
 	if (r == NULL)
-	    r = (char_u *)"latin1";
+	    r = (char_u *)ENC_DFLT;
 	return vim_strsave(r);
     }
 

src/option.c

@@ -430,14 +430,21 @@ set_init_1(int clean_arg)
 #  endif
 # endif
 
+# ifdef MSWIN
+    // MS-Windows has builtin support for conversion to and from Unicode, using
+    // "utf-8" for 'encoding' should work best for most users.
+    p = vim_strsave((char_u *)ENC_DFLT);
+# else
     // enc_locale() will try to find the encoding of the current locale.
+    // This works best for properly configured systems, old and new.
     p = enc_locale();
+# endif
     if (p != NULL)
     {
 	char_u *save_enc;
 
 	// Try setting 'encoding' and check if the value is valid.
-	// If not, go back to the default "latin1".
+	// If not, go back to the default encoding.
 	save_enc = p_enc;
 	p_enc = p;
 	if (STRCMP(p_enc, "gb18030") == 0)

src/option.h

@@ -127,7 +127,11 @@ typedef enum {
 #define ENC_UCSBOM	"ucs-bom"	// check for BOM at start of file
 
 // default value for 'encoding'
+#ifdef MSWIN
+# define ENC_DFLT	"utf-8"
+#else
 # define ENC_DFLT	"latin1"
+#endif
 
 // end-of-line style
 #define EOL_UNKNOWN	-1	// not defined yet

src/regexp.c

@@ -1279,6 +1279,7 @@ reg_match_visual(void)
     colnr_T	start, end;
     colnr_T	start2, end2;
     colnr_T	cols;
+    colnr_T	curswant;
 
     // Check if the buffer is the current buffer.
     if (rex.reg_buf != curbuf || VIsual.lnum == 0)
@@ -1297,6 +1298,7 @@ reg_match_visual(void)
 	    bot = VIsual;
 	}
 	mode = VIsual_mode;
+	curswant = wp->w_curswant;
     }
     else
     {
@@ -1311,6 +1313,7 @@ reg_match_visual(void)
 	    bot = curbuf->b_visual.vi_start;
 	}
 	mode = curbuf->b_visual.vi_mode;
+	curswant = curbuf->b_visual.vi_curswant;
     }
     lnum = rex.lnum + rex.reg_firstlnum;
     if (lnum < top.lnum || lnum > bot.lnum)
@@ -1331,7 +1334,7 @@ reg_match_visual(void)
 	    start = start2;
 	if (end2 > end)
 	    end = end2;
-	if (top.col == MAXCOL || bot.col == MAXCOL)
+	if (top.col == MAXCOL || bot.col == MAXCOL || curswant == MAXCOL)
 	    end = MAXCOL;
 	cols = win_linetabsize(wp, rex.line, (colnr_T)(rex.input - rex.line));
 	if (cols < start || cols > end - (*p_sel == 'e'))

src/term.c

@@ -4248,6 +4248,7 @@ add_termcode(char_u *name, char_u *string, int flags)
 	if (new_tc == NULL)
 	{
 	    tc_max_len -= 20;
+	    vim_free(s);
 	    return;
 	}
 	for (i = 0; i < tc_len; ++i)

src/terminal.c

@@ -445,6 +445,13 @@ term_start(
 
     if (check_restricted() || check_secure())
 	return NULL;
+#ifdef FEAT_CMDWIN
+    if (cmdwin_type != 0)
+    {
+	emsg(_(e_cannot_open_terminal_from_command_line_window));
+	return NULL;
+    }
+#endif
 
     if ((opt->jo_set & (JO_IN_IO + JO_OUT_IO + JO_ERR_IO))
 					 == (JO_IN_IO + JO_OUT_IO + JO_ERR_IO)

src/testdir/dumps/Test_cmdwin_no_terminal.dump

@@ -0,0 +1,12 @@
+| +0&#ffffff0@74
+|[+1&&|N|o| |N|a|m|e|]| @47|0|,|0|-|1| @9|A|l@1
+|:+0#4040ff13&|s+0#af5f00255&|e|t| +0#0000000&|c+0#e000e06&|m|d|h|e|i|g|h|t|=+0#0000000&|2| @58
+|:+0#4040ff13&> +0#0000000&@73
+|~+0#4040ff13&| @73
+|~| @73
+|~| @73
+|~| @73
+|~| @73
+|[+3#0000000&|C|o|m@1|a|n|d| |L|i|n|e|]| @42|2|,|0|-|1| @9|A|l@1
+|E+0#ffffff16#e000002|1@1|8@1|:| |C|a|n@1|o|t| |o|p|e|n| |a| |t|e|r|m|i|n|a|l| |f|r|o|m| |t|h|e| |c|o|m@1|a|n|d| |l|i|n|e| |w|i|n|d|o|w| +0#0000000#ffffff0@16
+@75

src/testdir/dumps/Test_hlsearch_block_visual_match.dump

@@ -0,0 +1,9 @@
+|a+0&#ffff4012@1| | +0&#ffffff0@56
+>b+0&#ffff4012@3| | +0&#ffffff0@54
+|c+0&#ffff4012@5| | +0&#ffffff0@52
+|~+0#4040ff13&| @58
+|~| @58
+|~| @58
+|~| @58
+|~| @58
+|/+0#0000000&|\|%|V| @37|2|,|1| @10|A|l@1| 

src/testdir/test_cmdline.vim

@@ -1192,6 +1192,21 @@ func Test_cmdwin_restore()
   call delete('XTest_restore')
 endfunc
 
+func Test_cmdwin_no_terminal()
+  CheckFeature cmdwin
+  CheckFeature terminal
+  CheckNotMSWindows
+
+  let buf = RunVimInTerminal('', {'rows': 12})
+  call TermWait(buf, 50)
+  call term_sendkeys(buf, ":set cmdheight=2\<CR>")
+  call term_sendkeys(buf, "q:")
+  call term_sendkeys(buf, ":let buf = term_start(['/bin/echo'], #{hidden: 1})\<CR>")
+  call VerifyScreenDump(buf, 'Test_cmdwin_no_terminal', {})
+  call term_sendkeys(buf, ":q\<CR>")
+  call StopVimInTerminal(buf)
+endfunc
+
 func Test_buffers_lastused()
   " check that buffers are sorted by time when wildmode has lastused
   call test_settime(1550020000)	  " middle

src/testdir/test_search.vim

@@ -984,6 +984,26 @@ func Test_hlsearch_and_visual()
   call delete('Xhlvisual_script')
 endfunc
 
+func Test_hlsearch_block_visual_match()
+  CheckScreendump
+
+  let lines =<< trim END
+    set hlsearch
+    call setline(1, ['aa', 'bbbb', 'cccccc'])
+  END
+  call writefile(lines, 'Xhlsearch_block')
+  let buf = RunVimInTerminal('-S Xhlsearch_block', {'rows': 9, 'cols': 60})
+
+  call term_sendkeys(buf, "G\<C-V>$kk\<Esc>")
+  sleep 100m
+  call term_sendkeys(buf, "/\\%V\<CR>")
+  sleep 100m
+  call VerifyScreenDump(buf, 'Test_hlsearch_block_visual_match', {})
+
+  call StopVimInTerminal(buf)
+  call delete('Xhlsearch_block')
+endfunc
+
 func Test_incsearch_substitute()
   CheckOption incsearch
 

src/testdir/test_startup.vim

@@ -276,6 +276,20 @@ func Test_V_arg()
    call assert_match("sourcing \"$VIMRUNTIME[\\/]defaults\.vim\"\r\nline 1: \" The default vimrc file\..*  verbose=15\n", out)
 endfunc
 
+" Test that an error is shown when the defaults.vim file could not be read
+" TODO: disabled - this causes ASAN errors for unknown reasons
+"func Test_defaults_error()
+"  " Can't catch the output of gvim.
+"  CheckNotGui
+"  CheckNotMSWindows
+"
+"  let out = system('VIMRUNTIME=/tmp ' .. GetVimCommand() .. ' --clean -cq')
+"  call assert_match("E1187: Failed to source defaults.vim", out)
+"
+"  let out = system('VIMRUNTIME=/tmp ' .. GetVimCommand() .. ' -u DEFAULTS -cq')
+"  call assert_match("E1187: Failed to source defaults.vim", out)
+"endfunc
+
 " Test the '-q [errorfile]' argument.
 func Test_q_arg()
   CheckFeature quickfix

src/testdir/test_writefile.vim

@@ -501,7 +501,7 @@ func Test_write_file_encoding()
   CheckMSWindows
   let save_encoding = &encoding
   let save_fileencodings = &fileencodings
-  set encoding& fileencodings&
+  set encoding=latin1 fileencodings&
   let text =<< trim END
     1 utf-8 text: Для Vim version 6.2.  Последнее изменение: 1970 Jan 01
     2 cp1251 text: <20><><EFBFBD> Vim version 6.2.  <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>: 1970 Jan 01

src/version.c

@@ -750,6 +750,22 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    2912,
+/**/
+    2911,
+/**/
+    2910,
+/**/
+    2909,
+/**/
+    2908,
+/**/
+    2907,
+/**/
+    2906,
+/**/
+    2905,
 /**/
     2904,
 /**/

src/vim.h

@@ -46,9 +46,6 @@
 #  endif
 # endif
 
-// for INT_MAX, LONG_MAX et al.
-# include <limits.h>
-
 /*
  * Cygwin may have fchdir() in a newer release, but in most versions it
  * doesn't work well and avoiding it keeps the binary backward compatible.
@@ -62,6 +59,9 @@
 # define UINT32_TYPEDEF uint32_t
 #endif
 
+// for INT_MAX, LONG_MAX et al.
+#include <limits.h>
+
 #if !defined(UINT32_TYPEDEF)
 # if defined(uint32_t)  // this doesn't catch typedefs, unfortunately
 #  define UINT32_TYPEDEF uint32_t