NVIM v0.3.8

FIXES:

5b47e4dc38 #10341 security: 'modeline', 'modelineexpr'
e4ecb70d08 #10345 Disallow API calls in the sandbox.

.travis.yml

@@ -74,7 +74,8 @@ jobs:
     - os: osx
       compiler: gcc
       osx_image: xcode9.4  # macOS 10.13
-    - os: linux
+    - if: branch = master
+      os: linux
       env: CI_TARGET=lint
     - stage: Flaky builds
       os: linux
@@ -126,6 +127,7 @@ addons:
 branches:
   only:
     - master
+    - /^release-\d+\.\d+$/
 
 cache:
   apt: true

CMakeLists.txt

@@ -112,7 +112,7 @@ set_property(CACHE CMAKE_BUILD_TYPE PROPERTY
 # version string, else they are combined with the result of `git describe`.
 set(NVIM_VERSION_MAJOR 0)
 set(NVIM_VERSION_MINOR 3)
-set(NVIM_VERSION_PATCH 2)
+set(NVIM_VERSION_PATCH 8)
 set(NVIM_VERSION_PRERELEASE "") # for package maintainers
 
 # API level

runtime/autoload/health/provider.vim

@@ -192,9 +192,9 @@ function! s:version_info(python) abort
   let nvim_path = s:trim(s:system([
         \ a:python, '-c',
         \ 'import sys; sys.path.remove(""); ' .
-        \ 'import pynvim; print(pynvim.__file__)']))
+        \ 'import neovim; print(neovim.__file__)']))
   if s:shell_error || empty(nvim_path)
-    return [python_version, 'unable to load pynvim Python module', pypi_version,
+    return [python_version, 'unable to load neovim Python module', pypi_version,
           \ nvim_path]
   endif
 
@@ -206,13 +206,13 @@ function! s:version_info(python) abort
     return a == b ? 0 : a > b ? 1 : -1
   endfunction
 
-  " Try to get pynvim.VERSION (added in 0.1.11dev).
+  " Try to get neovim.VERSION (added in 0.1.11dev).
   let nvim_version = s:system([a:python, '-c',
-        \ 'from pynvim import VERSION as v; '.
+        \ 'from neovim import VERSION as v; '.
         \ 'print("{}.{}.{}{}".format(v.major, v.minor, v.patch, v.prerelease))'],
         \ '', 1, 1)
   if empty(nvim_version)
-    let nvim_version = 'unable to find pynvim Python module version'
+    let nvim_version = 'unable to find neovim Python module version'
     let base = fnamemodify(nvim_path, ':h')
     let metas = glob(base.'-*/METADATA', 1, 1)
           \ + glob(base.'-*/PKG-INFO', 1, 1)
@@ -258,71 +258,56 @@ function! s:check_python(version) abort
   call health#report_start('Python ' . a:version . ' provider (optional)')
 
   let pyname = 'python'.(a:version == 2 ? '' : '3')
-  let pyenv = resolve(exepath('pyenv'))
-  let pyenv_root = exists('$PYENV_ROOT') ? resolve($PYENV_ROOT) : ''
+  let python_exe = ''
   let venv = exists('$VIRTUAL_ENV') ? resolve($VIRTUAL_ENV) : ''
   let host_prog_var = pyname.'_host_prog'
   let loaded_var = 'g:loaded_'.pyname.'_provider'
-  let python_bin = ''
   let python_multiple = []
 
   if exists(loaded_var) && !exists('*provider#'.pyname.'#Call')
     call health#report_info('Disabled ('.loaded_var.'='.eval(loaded_var).').  This might be due to some previous error.')
   endif
 
-  if !empty(pyenv)
-    if empty(pyenv_root)
-      call health#report_info(
-            \ 'pyenv was found, but $PYENV_ROOT is not set. `pyenv root` will be used.'
-            \ .' If you run into problems, try setting $PYENV_ROOT explicitly.'
-            \ )
-      let pyenv_root = s:trim(s:system([pyenv, 'root']))
-    endif
-
-    if !isdirectory(pyenv_root)
-      call health#report_error('Invalid pyenv root: '.pyenv_root)
-    else
-      call health#report_info(printf('pyenv: %s', pyenv))
-      call health#report_info(printf('pyenv root: %s', pyenv_root))
-    endif
-  endif
+  let [pyenv, pyenv_root] = s:check_for_pyenv()
 
   if exists('g:'.host_prog_var)
     call health#report_info(printf('Using: g:%s = "%s"', host_prog_var, get(g:, host_prog_var)))
   endif
 
-  let [pyname, pythonx_errs] = provider#pythonx#Detect(a:version)
+  let [pyname, pythonx_errors] = provider#pythonx#Detect(a:version)
+
   if empty(pyname)
-    call health#report_warn('No Python interpreter was found with the pynvim '
-            \ . 'module.  Using the first available for diagnostics.')
+    call health#report_warn('No Python executable found that can `import neovim`. '
+            \ . 'Using the first available executable for diagnostics.')
   elseif exists('g:'.host_prog_var)
-    let python_bin = pyname
+    let python_exe = pyname
   endif
 
-  if !empty(pythonx_errs)
-    call health#report_error('Python provider error', pythonx_errs)
+  " No Python executable could `import neovim`.
+  if !empty(pythonx_errors)
+    call health#report_error('Python provider error:', pythonx_errors)
 
-  elseif !empty(pyname) && empty(python_bin)
+  elseif !empty(pyname) && empty(python_exe)
     if !exists('g:'.host_prog_var)
       call health#report_info(printf('`g:%s` is not set.  Searching for '
             \ . '%s in the environment.', host_prog_var, pyname))
     endif
 
     if !empty(pyenv)
-      let python_bin = s:trim(s:system([pyenv, 'which', pyname], '', 1))
+      let python_exe = s:trim(s:system([pyenv, 'which', pyname], '', 1))
 
-      if empty(python_bin)
+      if empty(python_exe)
         call health#report_warn(printf('pyenv could not find %s.', pyname))
       endif
     endif
 
-    if empty(python_bin)
-      let python_bin = exepath(pyname)
+    if empty(python_exe)
+      let python_exe = exepath(pyname)
 
       if exists('$PATH')
         for path in split($PATH, has('win32') ? ';' : ':')
           let path_bin = s:normalize_path(path.'/'.pyname)
-          if path_bin != s:normalize_path(python_bin)
+          if path_bin != s:normalize_path(python_exe)
                 \ && index(python_multiple, path_bin) == -1
                 \ && executable(path_bin)
             call add(python_multiple, path_bin)
@@ -336,8 +321,8 @@ function! s:check_python(version) abort
                 \ . 'Set `g:%s` to avoid surprises.', pyname, host_prog_var))
         endif
 
-        if python_bin =~# '\<shims\>'
-          call health#report_warn(printf('`%s` appears to be a pyenv shim.', python_bin), [
+        if python_exe =~# '\<shims\>'
+          call health#report_warn(printf('`%s` appears to be a pyenv shim.', python_exe), [
                       \ '`pyenv` is not in $PATH, your pyenv installation is broken. '
                       \ .'Set `g:'.host_prog_var.'` to avoid surprises.',
                       \ ])
@@ -346,9 +331,9 @@ function! s:check_python(version) abort
     endif
   endif
 
-  if !empty(python_bin) && !exists('g:'.host_prog_var)
+  if !empty(python_exe) && !exists('g:'.host_prog_var)
     if empty(venv) && !empty(pyenv)
-          \ && !empty(pyenv_root) && resolve(python_bin) !~# '^'.pyenv_root.'/'
+          \ && !empty(pyenv_root) && resolve(python_exe) !~# '^'.pyenv_root.'/'
       call health#report_warn('pyenv is not set up optimally.', [
             \ printf('Create a virtualenv specifically '
             \ . 'for Neovim using pyenv, and set `g:%s`.  This will avoid '
@@ -362,7 +347,7 @@ function! s:check_python(version) abort
         let venv_root = fnamemodify(venv, ':h')
       endif
 
-      if resolve(python_bin) !~# '^'.venv_root.'/'
+      if resolve(python_exe) !~# '^'.venv_root.'/'
         call health#report_warn('Your virtualenv is not set up optimally.', [
               \ printf('Create a virtualenv specifically '
               \ . 'for Neovim and use `g:%s`.  This will avoid '
@@ -373,16 +358,16 @@ function! s:check_python(version) abort
     endif
   endif
 
-  if empty(python_bin) && !empty(pyname)
+  if empty(python_exe) && !empty(pyname)
     " An error message should have already printed.
     call health#report_error(printf('`%s` was not found.', pyname))
-  elseif !empty(python_bin) && !s:check_bin(python_bin)
-    let python_bin = ''
+  elseif !empty(python_exe) && !s:check_bin(python_exe)
+    let python_exe = ''
   endif
 
   " Check if $VIRTUAL_ENV is valid.
-  if exists('$VIRTUAL_ENV') && !empty(python_bin)
-    if $VIRTUAL_ENV ==# matchstr(python_bin, '^\V'.$VIRTUAL_ENV)
+  if exists('$VIRTUAL_ENV') && !empty(python_exe)
+    if $VIRTUAL_ENV ==# matchstr(python_exe, '^\V'.$VIRTUAL_ENV)
       call health#report_info('$VIRTUAL_ENV matches executable')
     else
       call health#report_warn(
@@ -393,7 +378,7 @@ function! s:check_python(version) abort
   endif
 
   " Diagnostic output
-  call health#report_info('Executable: ' . (empty(python_bin) ? 'Not found' : python_bin))
+  call health#report_info('Executable: ' . (empty(python_exe) ? 'Not found' : python_exe))
   if len(python_multiple)
     for path_bin in python_multiple
       call health#report_info('Other python executable: ' . path_bin)
@@ -402,30 +387,39 @@ function! s:check_python(version) abort
 
   let pip = 'pip' . (a:version == 2 ? '' : '3')
 
-  if !empty(python_bin)
-    let [pyversion, current, latest, status] = s:version_info(python_bin)
+  if empty(python_exe)
+    " No Python executable can import 'neovim'. Check if any Python executable
+    " can import 'pynvim'. If so, that Python failed to import 'neovim' as
+    " well, which is most probably due to a failed pip upgrade:
+    " https://github.com/neovim/neovim/wiki/Following-HEAD#20181118
+    let [pynvim_exe, errors] = provider#pythonx#DetectByModule('pynvim', a:version)
+    if !empty(pynvim_exe)
+      call health#report_error(
+            \ 'Detected pip upgrade failure: Python executable can import "pynvim" but '
+            \ . 'not "neovim": '. pynvim_exe,
+            \ "Use that Python version to reinstall \"pynvim\" and optionally \"neovim\".\n"
+            \ . pip ." uninstall pynvim neovim\n"
+            \ . pip ." install pynvim\n"
+            \ . pip ." install neovim  # only if needed by third-party software")
+    endif
+  else
+    let [pyversion, current, latest, status] = s:version_info(python_exe)
+
     if a:version != str2nr(pyversion)
       call health#report_warn('Unexpected Python version.' .
                   \ ' This could lead to confusing error messages.')
     endif
+
     if a:version == 3 && str2float(pyversion) < 3.3
       call health#report_warn('Python 3.3+ is recommended.')
     endif
 
     call health#report_info('Python version: ' . pyversion)
+
     if s:is_bad_response(status)
       call health#report_info(printf('pynvim version: %s (%s)', current, status))
     else
       call health#report_info(printf('pynvim version: %s', current))
-      let [module_found, _msg] = provider#pythonx#CheckForModule(python_bin,
-            \ 'neovim', a:version)
-      if !module_found
-        call health#report_error('Importing "neovim" failed.',
-              \ "Reinstall \"pynvim\" and optionally \"neovim\" packages.\n" .
-              \    pip ." uninstall pynvim neovim\n" .
-              \    pip ." install pynvim\n" .
-              \    pip ." install neovim # only if needed by third-party software")
-      endif
     endif
 
     if s:is_bad_response(current)
@@ -443,7 +437,37 @@ function! s:check_python(version) abort
       call health#report_ok(printf('Latest pynvim is installed.'))
     endif
   endif
+endfunction
 
+" Check if pyenv is available and a valid pyenv root can be found, then return
+" their respective paths. If either of those is invalid, return two empty
+" strings, effectivly ignoring pyenv.
+function! s:check_for_pyenv() abort
+  let pyenv_path = resolve(exepath('pyenv'))
+
+  if empty(pyenv_path)
+    return ['', '']
+  endif
+
+  call health#report_info('pyenv: Path: '. pyenv_path)
+
+  let pyenv_root = exists('$PYENV_ROOT') ? resolve($PYENV_ROOT) : ''
+
+  if empty(pyenv_root)
+    let pyenv_root = s:trim(s:system([pyenv_path, 'root']))
+    call health#report_info('pyenv: $PYENV_ROOT is not set. Infer from `pyenv root`.')
+  endif
+
+  if !isdirectory(pyenv_root)
+    call health#report_warn(
+          \ printf('pyenv: Root does not exist: %s. '
+          \ . 'Ignoring pyenv for all following checks.', pyenv_root))
+    return ['', '']
+  endif
+
+  call health#report_info('pyenv: Root: '.pyenv_root)
+
+  return [pyenv_path, pyenv_root]
 endfunction
 
 function! s:check_ruby() abort

runtime/autoload/provider/pythonx.vim

@@ -10,7 +10,7 @@ function! provider#pythonx#Require(host) abort
 
   " Python host arguments
   let prog = (ver == '2' ?  provider#python#Prog() : provider#python3#Prog())
-  let args = [prog, '-c', 'import sys; sys.path.remove(""); import pynvim; pynvim.start_host()']
+  let args = [prog, '-c', 'import sys; sys.path.remove(""); import neovim; neovim.start_host()']
 
   " Collect registered Python plugins into args
   let python_plugins = remote#host#PluginsForHost(a:host.name)
@@ -21,37 +21,45 @@ function! provider#pythonx#Require(host) abort
   return provider#Poll(args, a:host.orig_name, '$NVIM_PYTHON_LOG_FILE')
 endfunction
 
-function! provider#pythonx#Detect(major_ver) abort
-  if a:major_ver == 2
-    if exists('g:python_host_prog')
-      return [expand(g:python_host_prog), '']
-    else
-      let progs = ['python2', 'python2.7', 'python2.6', 'python']
-    endif
-  else
-    if exists('g:python3_host_prog')
-      return [expand(g:python3_host_prog), '']
-    else
-      let progs = ['python3', 'python3.7', 'python3.6', 'python3.5',
-            \ 'python3.4', 'python3.3', 'python']
-    endif
+function! s:get_python_executable_from_host_var(major_version) abort
+  return expand(get(g:, 'python'.(a:major_version == 3 ? '3' : '').'_host_prog', ''))
+endfunction
+
+function! s:get_python_candidates(major_version) abort
+  return {
+        \ 2: ['python2', 'python2.7', 'python2.6', 'python'],
+        \ 3: ['python3', 'python3.7', 'python3.6', 'python3.5', 'python3.4', 'python3.3',
+        \     'python']
+        \ }[a:major_version]
+endfunction
+
+" Returns [path_to_python_executable, error_message]
+function! provider#pythonx#Detect(major_version) abort
+  return provider#pythonx#DetectByModule('neovim', a:major_version)
+endfunction
+
+" Returns [path_to_python_executable, error_message]
+function! provider#pythonx#DetectByModule(module, major_version) abort
+  let python_exe = s:get_python_executable_from_host_var(a:major_version)
+
+  if !empty(python_exe)
+    return [python_exe, '']
   endif
 
+  let candidates = s:get_python_candidates(a:major_version)
   let errors = []
 
-  for prog in progs
-    let [result, err] = provider#pythonx#CheckForModule(prog, 'pynvim', a:major_ver)
+  for exe in candidates
+    let [result, error] = provider#pythonx#CheckForModule(exe, a:module, a:major_version)
     if result
-      return [prog, err]
+      return [exe, error]
     endif
-    " Accumulate errors in case we don't find
-    " any suitable Python interpreter.
-    call add(errors, err)
+    " Accumulate errors in case we don't find any suitable Python executable.
+    call add(errors, error)
   endfor
 
-  " No suitable Python interpreter found.
-  return ['', 'provider/pythonx: Could not load Python ' . a:major_ver
-        \ . ":\n" .  join(errors, "\n")]
+  " No suitable Python executable found.
+  return ['', 'provider/pythonx: Could not load Python '.a:major_version.":\n".join(errors, "\n")]
 endfunction
 
 " Returns array: [prog_exitcode, prog_version]
@@ -75,12 +83,12 @@ function! provider#pythonx#CheckForModule(prog, module, major_version) abort
 
   let min_version = (a:major_version == 2) ? '2.6' : '3.3'
 
-  " Try to load pynvim module, and output Python version.
+  " Try to load module, and output Python version.
   " Exit codes:
-  "   0  pynvim module can be loaded.
-  "   2  pynvim module cannot be loaded.
+  "   0  module can be loaded.
+  "   2  module cannot be loaded.
   "   Otherwise something else went wrong (e.g. 1 or 127).
-  let [prog_exitcode, prog_version] = s:import_module(a:prog, 'pynvim')
+  let [prog_exitcode, prog_version] = s:import_module(a:prog, a:module)
 
   if prog_exitcode == 2 || prog_exitcode == 0
     " Check version only for expected return codes.
@@ -94,7 +102,7 @@ function! provider#pythonx#CheckForModule(prog, module, major_version) abort
   endif
 
   if prog_exitcode == 2
-    return [0, prog_path.' does not have the "pynvim" module. :help provider-python']
+    return [0, prog_path.' does not have the "' . a:module . '" module. :help provider-python']
   elseif prog_exitcode == 127
     " This can happen with pyenv's shims.
     return [0, prog_path . ' does not exist: ' . prog_version]

runtime/doc/options.txt

@@ -478,14 +478,17 @@ backslash in front of the ':' will be removed.  Example:
    /* vi:set dir=c\:\tmp: */ ~
 This sets the 'dir' option to "c:\tmp".  Only a single backslash before the
 ':' is removed.  Thus to include "\:" you have to specify "\\:".
-
+							*E992*
 No other commands than "set" are supported, for security reasons (somebody
 might create a Trojan horse text file with modelines).  And not all options
-can be set.  For some options a flag is set, so that when it's used the
-|sandbox| is effective.  Still, there is always a small risk that a modeline
-causes trouble.  E.g., when some joker sets 'textwidth' to 5 all your lines
-are wrapped unexpectedly.  So disable modelines before editing untrusted text.
-The mail ftplugin does this, for example.
+can be set.  For some options a flag is set, so that when the value is used
+the |sandbox| is effective.  Some options can only be set from the modeline
+when 'modelineexpr' is set (the default is off).
+
+Still, there is always a small risk that a modeline causes trouble.  E.g.,
+when some joker sets 'textwidth' to 5 all your lines are wrapped unexpectedly.
+So disable modelines before editing untrusted text.  The mail ftplugin does
+this, for example.
 
 Hint: If you would like to do something else than setting an option, you could
 define an autocommand that checks the file for a specific string.  For
@@ -2439,7 +2442,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 	The expression will be evaluated in the |sandbox| if set from a
 	modeline, see |sandbox-option|.
 	This option can't be set from a |modeline| when the 'diff' option is
-	on.
+	on or the 'modelineexpr' option is off.
 
 	It is not allowed to change text or jump to another window while
 	evaluating 'foldexpr' |textlock|.
@@ -2554,6 +2557,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 
 	The expression will be evaluated in the |sandbox| if set from a
 	modeline, see |sandbox-option|.
+	This option cannot be set in a modeline when 'modelineexpr' is off.
 
 	It is not allowed to change text or jump to another window while
 	evaluating 'foldtext' |textlock|.
@@ -2589,16 +2593,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 	The expression will be evaluated in the |sandbox| when set from a
 	modeline, see |sandbox-option|.  That stops the option from working,
 	since changing the buffer text is not allowed.
-
-					*'formatoptions'* *'fo'*
-'formatoptions' 'fo'	string (default: "tcqj", Vi default: "vt")
-			local to buffer
-	This is a sequence of letters which describes how automatic
-	formatting is to be done.  See |fo-table|.  When the 'paste' option is
-	on, no formatting is done (like 'formatoptions' is empty).  Commas can
-	be inserted for readability.
-	To avoid problems with flags that are added in the future, use the
-	"+=" and "-=" feature of ":set" |add-option-flags|.
+	This option cannot be set in a modeline when 'modelineexpr' is off.
+	NOTE: This option is set to "" when 'compatible' is set.
 
 					*'formatlistpat'* *'flp'*
 'formatlistpat' 'flp'	string (default: "^\s*\d\+[\]:.)}\t ]\s*")
@@ -2613,6 +2609,16 @@ A jump table for the options with a short description can be found at |Q_op|.
 	The default recognizes a number, followed by an optional punctuation
 	character and white space.
 
+					*'formatoptions'* *'fo'*
+'formatoptions' 'fo'	string (default: "tcqj", Vi default: "vt")
+			local to buffer
+	This is a sequence of letters which describes how automatic
+	formatting is to be done.  See |fo-table|.  When the 'paste' option is
+	on, no formatting is done (like 'formatoptions' is empty).  Commas can
+	be inserted for readability.
+	To avoid problems with flags that are added in the future, use the
+	"+=" and "-=" feature of ":set" |add-option-flags|.
+
 						*'formatprg'* *'fp'*
 'formatprg' 'fp'	string (default "")
 			global or local to buffer |global-local|
@@ -2643,6 +2649,9 @@ A jump table for the options with a short description can be found at |Q_op|.
 	- system signals low battery life
 	- Nvim exits abnormally
 
+	This option cannot be set from a |modeline| or in the |sandbox|, for
+	security reasons.
+
 				   *'gdefault'* *'gd'* *'nogdefault'* *'nogd'*
 'gdefault' 'gd'		boolean	(default off)
 			global
@@ -2978,6 +2987,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 	'guitabtooltip' is used for the tooltip, see below.
 	The expression will be evaluated in the |sandbox| when set from a
 	modeline, see |sandbox-option|.
+	This option cannot be set in a modeline when 'modelineexpr' is off.
 
 	Only used when the GUI tab pages line is displayed.  'e' must be
 	present in 'guioptions'.  For the non-GUI tab pages line 'tabline' is
@@ -3106,6 +3116,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 	When this option contains printf-style '%' items, they will be
 	expanded according to the rules used for 'statusline'.  See
 	'titlestring' for example settings.
+	This option cannot be set in a modeline when 'modelineexpr' is off.
 
 			*'ignorecase'* *'ic'* *'noignorecase'* *'noic'*
 'ignorecase' 'ic'	boolean	(default off)
@@ -3209,6 +3220,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 
 	The expression will be evaluated in the |sandbox| when set from a
 	modeline, see |sandbox-option|.
+	This option cannot be set in a modeline when 'modelineexpr' is off.
 
 	It is not allowed to change text or jump to another window while
 	evaluating 'includeexpr' |textlock|.
@@ -3277,6 +3289,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 
 	The expression will be evaluated in the |sandbox| when set from a
 	modeline, see |sandbox-option|.
+	This option cannot be set in a modeline when 'modelineexpr' is off.
 
 	It is not allowed to change text or jump to another window while
 	evaluating 'indentexpr' |textlock|.
@@ -3879,10 +3892,23 @@ A jump table for the options with a short description can be found at |Q_op|.
 <	If you have less than 512 Mbyte |:mkspell| may fail for some
 	languages, no matter what you set 'mkspellmem' to.
 
+	This option cannot be set from a |modeline| or in the |sandbox|.
+
 				   *'modeline'* *'ml'* *'nomodeline'* *'noml'*
 'modeline' 'ml'		boolean	(Vim default: on (off for root),
 				 Vi default: off)
 			local to buffer
+	If 'modeline' is on 'modelines' gives the number of lines that is
+	checked for set commands.  If 'modeline' is off or 'modelines' is zero
+	no lines are checked.  See |modeline|.
+
+			   *'modelineexpr'* *'mle'* *'nomodelineexpr'* *'nomle'*
+'modelineexpr' 'mle'	boolean (default: off)
+			global
+	When on allow some options that are an expression to be set in the
+	modeline.  Check the option for whether it is affected by
+	'modelineexpr'.  Also see |modeline|.
+
 						*'modelines'* *'mls'*
 'modelines' 'mls'	number	(default 5)
 			global
@@ -4622,6 +4648,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 	When this option is not empty, it determines the content of the ruler
 	string, as displayed for the 'ruler' option.
 	The format of this option is like that of 'statusline'.
+	This option cannot be set in a modeline when 'modelineexpr' is off.
+
 	The default ruler width is 17 characters.  To make the ruler 15
 	characters wide, put "%15(" at the start and "%)" at the end.
 	Example: >
@@ -5252,7 +5280,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 		"Pattern not found", "Back at original", etc.
 	  q	use "recording" instead of "recording @a"
 	  F	don't give the file info when editing a file, like `:silent`
-		was used for the command
+		was used for the command; note that this also affects messages
+		from autocommands
 
 	This gives you the opportunity to avoid that a change between buffers
 	requires you to hit <Enter>, but still gives as useful a message as
@@ -5527,7 +5556,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 
 	After this option has been set successfully, Vim will source the files
 	"spell/LANG.vim" in 'runtimepath'.  "LANG" is the value of 'spelllang'
-	up to the first comma, dot or underscore.
+	up to the first character that is not an ASCII letter and not a dash.
 	Also see |set-spc-auto|.
 
 
@@ -5773,6 +5802,7 @@ A jump table for the options with a short description can be found at |Q_op|.
 
 	The 'statusline' option will be evaluated in the |sandbox| if set from
 	a modeline, see |sandbox-option|.
+	This option cannot be set in a modeline when 'modelineexpr' is off.
 
 	It is not allowed to change text or jump to another window while
 	evaluating 'statusline' |textlock|.
@@ -5927,6 +5957,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 	the text to be displayed.  Use "%1T" for the first label, "%2T" for
 	the second one, etc.  Use "%X" items for closing labels.
 
+	This option cannot be set in a modeline when 'modelineexpr' is off.
+
 	Keep in mind that only one of the tab pages is the current one, others
 	are invisible and you can't jump to their windows.
 
@@ -6203,8 +6235,11 @@ A jump table for the options with a short description can be found at |Q_op|.
 			global
 	When this option is not empty, it will be used for the title of the
 	window.  This happens only when the 'title' option is on.
+
 	When this option contains printf-style '%' items, they will be
 	expanded according to the rules used for 'statusline'.
+	This option cannot be set in a modeline when 'modelineexpr' is off.
+
 	Example: >
 	    :auto BufEnter * let &titlestring = hostname() . "/" . expand("%:p")
 	    :set title titlestring=%<%F%=%l/%L-%P titlelen=70
@@ -6238,6 +6273,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 	undo file that exists is used.  When it cannot be read an error is
 	given, no further entry is used.
 	See |undo-persistence|.
+	This option cannot be set from a |modeline| or in the |sandbox|, for
+	security reasons.
 
 				*'undofile'* *'noundofile'* *'udf'* *'noudf'*
 'undofile' 'udf'	boolean	(default off)

runtime/doc/provider.txt

@@ -24,29 +24,34 @@ Nvim supports Python |remote-plugin|s and the Vim legacy |python2| and
 |python3| interfaces (which are implemented as remote-plugins).
 Note: Only the Vim 7.3 API is supported; bindeval (Vim 7.4) is not.
 
+
 PYTHON QUICKSTART ~
 
-Install the "pynvim" Python package:
+To use Python plugins, you need the "pynvim" module. Run |:checkhealth| to see
+if you already have it (some package managers install the module with Nvim
+itself).
 
-- Run |:checkhealth| to see if you already have the package (some package
-  managers install the "pynvim" Python package with Nvim itself).
+For Python 3 plugins:
+1. Make sure Python 3.4+ is available in your $PATH.
+2. Install the module (try "pip" if "pip3" is missing): >
+   pip3 install --user --upgrade pynvim
 
-- For Python 2 plugins, make sure Python 2.7 is available in your $PATH, then
-  install the package systemwide: >
-    sudo pip2 install --upgrade pynvim
-<  or for the current user: >
-    pip2 install --user --upgrade pynvim
-<  If "pip2" is missing, try "pip".
+For Python 2 plugins:
+1. Make sure Python 2.7 is available in your $PATH.
+2. Install the module (try "pip" if "pip2" is missing): >
+   pip2 install --user --upgrade pynvim
 
-- For Python 3 plugins, make sure Python 3.4+ is available in your $PATH, then
-  install the package systemwide: >
-    sudo pip3 install --upgrade pynvim
-<  or for the current user: >
-    pip3 install --user --upgrade pynvim
-<  If "pip3" is missing, try "pip".
+The pip `--upgrade` flag ensures that you get the latest version even if
+a previous version was already installed.
+
+See also |python-virtualenv|.
+
+Note: The old "neovim" module was renamed to "pynvim".
+https://github.com/neovim/neovim/wiki/Following-HEAD#20181118
+If you run into problems, uninstall _both_ then install "pynvim" again: >
+  pip uninstall neovim pynvim
+  pip install pynvim
 
-- The `--upgrade` flag ensures you have the latest version even if a previous
-  version was already installed.
 
 PYTHON PROVIDER CONFIGURATION ~
 						*g:python_host_prog*
@@ -67,8 +72,9 @@ To disable Python 2 support: >
 To disable Python 3 support: >
     let g:loaded_python3_provider = 1
 
-PYTHON VIRTUALENVS ~
 
+PYTHON VIRTUALENVS ~
+						*python-virtualenv*
 If you plan to use per-project virtualenvs often, you should assign one
 virtualenv for Neovim and hard-code the interpreter path via
 |g:python3_host_prog| (or |g:python_host_prog|) so that the "pynvim" package
@@ -91,6 +97,7 @@ Ruby integration		    	      *provider-ruby*
 Nvim supports Ruby |remote-plugin|s and the Vim legacy |ruby-vim| interface
 (which is itself implemented as a Nvim remote-plugin).
 
+
 RUBY QUICKSTART ~
 
 To use Ruby plugins with Nvim, install the latest "neovim" RubyGem: >
@@ -98,6 +105,7 @@ To use Ruby plugins with Nvim, install the latest "neovim" RubyGem: >
 
 Run |:checkhealth| to see if your system is up-to-date.
 
+
 RUBY PROVIDER CONFIGURATION ~
 						*g:loaded_ruby_provider*
 To disable Ruby support: >
@@ -120,6 +128,7 @@ Node.js integration				*provider-nodejs*
 Nvim supports Node.js |remote-plugin|s.
 https://github.com/neovim/node-client/
 
+
 NODEJS QUICKSTART~
 
 To use javascript remote-plugins with Nvim, install the "neovim" npm package: >
@@ -127,6 +136,7 @@ To use javascript remote-plugins with Nvim, install the "neovim" npm package: >
 
 Run |:checkhealth| to see if your system is up-to-date.
 
+
 NODEJS PROVIDER CONFIGURATION~
 						*g:loaded_node_provider*
 To disable Node.js support: >

scripts/genappimage.sh

@@ -37,29 +37,24 @@ VERSION=$("$ROOT_DIR"/build/bin/nvim --version | head -n 1 | grep -o 'v.*')
 
 cd "$APP_BUILD_DIR"
 
-curl -Lo "$APP_BUILD_DIR"/appimage_functions.sh https://github.com/AppImage/AppImages/raw/master/functions.sh
-. ./appimage_functions.sh
+# Only downloads linuxdeploy if the remote file is different from local
+if [ -e "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage ]; then
+  curl -Lo "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage \
+    -z "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage \
+    https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/linuxdeploy-x86_64.AppImage  
+else
+  curl -Lo "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage \
+    https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/linuxdeploy-x86_64.AppImage
+fi
 
-# Copy desktop and icon file to AppDir for AppRun to pick them up.
-# get_apprun
-# get_desktop
-cp "$ROOT_DIR/runtime/nvim.desktop" "$APP_DIR/"
-cp "$ROOT_DIR/runtime/nvim.png" "$APP_DIR/"
+chmod +x "$APP_BUILD_DIR"/linuxdeploy-x86_64.AppImage
+
+# metainfo is not packaged automatically by linuxdeploy
 mkdir "$APP_DIR/usr/share/metainfo/"
 cp "$ROOT_DIR/runtime/nvim.appdata.xml" "$APP_DIR/usr/share/metainfo/"
 
 cd "$APP_DIR"
 
-# copy dependencies
-copy_deps
-# Move the libraries to usr/bin
-move_lib
-
-# Delete stuff that should not go into the AppImage.
-# Delete dangerous libraries; see
-# https://github.com/AppImage/AppImages/blob/master/excludelist
-delete_blacklisted
-
 ########################################################################
 # AppDir complete. Now package it as an AppImage.
 ########################################################################
@@ -78,21 +73,24 @@ chmod 755 AppRun
 
 cd "$APP_BUILD_DIR" # Get out of AppImage directory.
 
+# Set the name of the file generated by appimage
+export OUTPUT=nvim.appimage
+
+# If it's a release generate the zsync file
+if [ -n "$TAG" ]; then
+  export UPDATE_INFORMATION="gh-releases-zsync|neovim|neovim|$TAG|nvim.appimage.zsync"
+fi
+
 # Generate AppImage.
 #   - Expects: $ARCH, $APP, $VERSION env vars
 #   - Expects: ./$APP.AppDir/ directory
-#   - Produces: ../out/$APP-$VERSION.glibc$GLIBC_NEEDED-$ARCH.AppImage
-if [ -n "$TAG" ]; then
-  generate_type2_appimage -u "gh-releases-zsync|neovim|neovim|$TAG|nvim.appimage.zsync"
-else
-  generate_type2_appimage
-fi
+#   - Produces: ./nvim.appimage
+./linuxdeploy-x86_64.AppImage --appdir $APP.AppDir -d $ROOT_DIR/runtime/nvim.desktop -i \
+"$ROOT_DIR/runtime/nvim.png" --output appimage
 
 # Moving the final executable to a different folder so it isn't in the
 # way for a subsequent build.
 
-mv "$ROOT_DIR"/out/*.AppImage* "$ROOT_DIR"/build/bin
-# Remove the (now empty) folder the AppImage was built in
-rmdir "$ROOT_DIR"/out
+mv "$ROOT_DIR"/build/nvim.appimage* "$ROOT_DIR"/build/bin
 
 echo 'genappimage.sh: finished'

scripts/git-log-pretty-since.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 
-# Shows a log with changes grouped next to their merge-commit.
+# Prints a nicely-formatted commit history.
+#   - Commits are grouped below their merge-commit.
+#   - Issue numbers are moved next to the commit-id.
 #
 # Parameters:
 #   $1    "since" commit
@@ -19,14 +21,32 @@ is_merge_commit() {
   git log $1^2 >/dev/null 2>&1 && return 0 || return 1
 }
 
+# Removes parens from issue/ticket/PR numbers.
+#
+# Example:
+#   in:   3340e08becbf foo (#9423)
+#   out:  3340e08becbf foo #9423
+_deparen() {
+  sed 's/(\(\#[0-9]\{3,\}\))/\1/g'
+}
+
+# Cleans up issue/ticket/PR numbers in the commit descriptions.
+#
+# Example:
+#   in:   3340e08becbf foo (#9423)
+#   out:  3340e08becbf #9423 foo
+_format_ticketnums() {
+  nvim -Es +'g/\v(#[0-9]{3,})/norm! ngEldE0ep' +'%p' | _deparen
+}
+
 for commit in $(git log --format='%H' --first-parent "$__SINCE"..HEAD); do
   if is_merge_commit ${commit} ; then
       if [ -z "$__INVMATCH" ] || ! git log --oneline ${commit}^1..${commit}^2 \
-           | grep -E "$__INVMATCH" >/dev/null 2>&1 ; then
+           | >/dev/null 2>&1 grep -E "$__INVMATCH" ; then
         git log -1 --oneline ${commit}
         git log --format='    %h %s' ${commit}^1..${commit}^2
       fi
   else
     git log -1 --oneline ${commit}
   fi
-done
+done | _format_ticketnums

scripts/release.sh

@@ -15,10 +15,6 @@
 #     - Tag the commit.
 #   Create the "version bump" commit:
 #     - CMakeLists.txt: Set NVIM_VERSION_PRERELEASE to "-dev"
-#
-# Manual steps:
-#   - CMakeLists.txt: Bump NVIM_VERSION_* as appropriate.
-#   - git push --follow-tags
 
 set -e
 set -u
@@ -67,7 +63,7 @@ _do_release_commit() {
 
   if ! test "$ARG1" = '--use-current-commit' ; then
     echo "Building changelog since ${__LAST_TAG}..."
-    __CHANGELOG="$(./scripts/git-log-pretty-since.sh "$__LAST_TAG" 'vim-patch:\S')"
+    __CHANGELOG="$(./scripts/git-log-pretty-since.sh "$__LAST_TAG" 'vim-patch:[^[:space:]]')"
 
     git add CMakeLists.txt
     git commit --edit -m "${__RELEASE_MSG} ${__CHANGELOG}"
@@ -95,5 +91,9 @@ _do_bump_commit
 echo "
 Next steps:
     - Double-check NVIM_VERSION_* in CMakeLists.txt
-    - git push --follow-tags
-    - update website: index.html"
+    - Push the tag:
+        git push --follow-tags
+    - Update the 'stable' tag:
+        git push --force upstream HEAD^:refs/tags/stable
+        git fetch --tags
+    - Update website: index.html"

src/nvim/api/vim.c

@@ -594,7 +594,7 @@ void nvim_set_current_dir(String dir, Error *err)
     return;
   }
 
-  post_chdir(kCdScopeGlobal);
+  post_chdir(kCdScopeGlobal, true);
   try_end(err);
 }
 

src/nvim/buffer.c

@@ -1559,6 +1559,7 @@ void do_autochdir(void)
     if (starting == 0
         && curbuf->b_ffname != NULL
         && vim_chdirfile(curbuf->b_ffname) == OK) {
+      post_chdir(kCdScopeGlobal, false);
       shorten_fnames(true);
     }
   }
@@ -4923,9 +4924,15 @@ chk_modeline (
     *e = NUL;                         /* truncate the set command */
 
     if (*s != NUL) {                  /* skip over an empty "::" */
+      const int secure_save = secure;
       save_SID = current_SID;
       current_SID = SID_MODELINE;
+      // Make sure no risky things are executed as a side effect.
+      secure = 1;
+
       retval = do_set(s, OPT_MODELINE | OPT_LOCAL | flags);
+
+      secure = secure_save;
       current_SID = save_SID;
       if (retval == FAIL)                     /* stop if error found */
         break;

src/nvim/eval.c

@@ -241,13 +241,14 @@ typedef enum {
                                   ///< the value (prevents error message).
 } GetLvalFlags;
 
-// function flags
+// flags used in uf_flags
 #define FC_ABORT    0x01          // abort function on error
 #define FC_RANGE    0x02          // function accepts range
 #define FC_DICT     0x04          // Dict function, uses "self"
 #define FC_CLOSURE  0x08          // closure, uses outer scope variables
 #define FC_DELETED  0x10          // :delfunction used while uf_refcount > 0
 #define FC_REMOVED  0x20          // function redefined while uf_refcount > 0
+#define FC_SANDBOX  0x40          // function defined in the sandbox
 
 // The names of packages that once were loaded are remembered.
 static garray_T ga_loaded = { 0, 0, sizeof(char_u *), 4, NULL };
@@ -5853,6 +5854,9 @@ static int get_lambda_tv(char_u **arg, typval_T *rettv, bool evaluate)
     if (prof_def_func()) {
       func_do_profile(fp);
     }
+    if (sandbox) {
+      flags |= FC_SANDBOX;
+    }
     fp->uf_varargs = true;
     fp->uf_flags = flags;
     fp->uf_calls = 0;
@@ -6512,6 +6516,10 @@ static void float_op_wrapper(typval_T *argvars, typval_T *rettv, FunPtr fptr)
 
 static void api_wrapper(typval_T *argvars, typval_T *rettv, FunPtr fptr)
 {
+  if (check_restricted() || check_secure()) {
+    return;
+  }
+
   ApiDispatchWrapper fn = (ApiDispatchWrapper)fptr;
 
   Array args = ARRAY_DICT_INIT;
@@ -20315,6 +20323,9 @@ void ex_function(exarg_T *eap)
   if (prof_def_func())
     func_do_profile(fp);
   fp->uf_varargs = varargs;
+  if (sandbox) {
+    flags |= FC_SANDBOX;
+  }
   fp->uf_flags = flags;
   fp->uf_calls = 0;
   fp->uf_script_ID = current_SID;
@@ -21305,6 +21316,7 @@ void call_user_func(ufunc_T *fp, int argcount, typval_T *argvars,
   char_u      *save_sourcing_name;
   linenr_T save_sourcing_lnum;
   scid_T save_current_SID;
+  bool using_sandbox = false;
   funccall_T  *fc;
   int save_did_emsg;
   static int depth = 0;
@@ -21462,6 +21474,12 @@ void call_user_func(ufunc_T *fp, int argcount, typval_T *argvars,
   save_sourcing_name = sourcing_name;
   save_sourcing_lnum = sourcing_lnum;
   sourcing_lnum = 1;
+
+  if (fp->uf_flags & FC_SANDBOX) {
+    using_sandbox = true;
+    sandbox++;
+  }
+
   // need space for new sourcing_name:
   // * save_sourcing_name
   // * "["number"].." or "function "
@@ -21622,6 +21640,9 @@ void call_user_func(ufunc_T *fp, int argcount, typval_T *argvars,
   if (do_profiling_yes) {
     script_prof_restore(&wait_start);
   }
+  if (using_sandbox) {
+    sandbox--;
+  }
 
   if (p_verbose >= 12 && sourcing_name != NULL) {
     ++no_wait_return;
@@ -22683,6 +22704,16 @@ static void script_host_eval(char *name, typval_T *argvars, typval_T *rettv)
 
 typval_T eval_call_provider(char *provider, char *method, list_T *arguments)
 {
+  if (!eval_has_provider(provider)) {
+    emsgf("E319: No \"%s\" provider found. Run \":checkhealth provider\"",
+          provider);
+    return (typval_T){
+      .v_type = VAR_NUMBER,
+      .v_lock = VAR_UNLOCKED,
+      .vval.v_number = (varnumber_T)0
+    };
+  }
+
   char func[256];
   int name_len = snprintf(func, sizeof(func), "provider#%s#Call", provider);
 

src/nvim/ex_cmds.lua

@@ -1004,7 +1004,7 @@ return {
   },
   {
     command='function',
-    flags=bit.bor(EXTRA, BANG, CMDWIN),
+    flags=bit.bor(EXTRA, BANG, SBOXOK, CMDWIN),
     addr_type=ADDR_LINES,
     func='ex_function',
   },

src/nvim/ex_cmds2.c

@@ -3908,12 +3908,7 @@ static void script_host_execute(char *name, exarg_T *eap)
     tv_list_append_number(args, (int)eap->line1);
     tv_list_append_number(args, (int)eap->line2);
 
-    if (!eval_has_provider(name)) {
-      emsgf("E319: No \"%s\" provider found. Run \":checkhealth provider\"",
-            name);
-    } else {
-      (void)eval_call_provider(name, "execute", args);
-    }
+    (void)eval_call_provider(name, "execute", args);
   }
 }
 

src/nvim/ex_docmd.c

@@ -7218,7 +7218,7 @@ void free_cd_dir(void)
 /// Deal with the side effects of changing the current directory.
 ///
 /// @param scope  Scope of the function call (global, tab or window).
-void post_chdir(CdScope scope)
+void post_chdir(CdScope scope, bool trigger_dirchanged)
 {
   // Always overwrite the window-local CWD.
   xfree(curwin->w_localdir);
@@ -7258,7 +7258,10 @@ void post_chdir(CdScope scope)
   }
 
   shorten_fnames(true);
-  do_autocmd_dirchanged(cwd, scope);
+
+  if (trigger_dirchanged) {
+    do_autocmd_dirchanged(cwd, scope);
+  }
 }
 
 /// `:cd`, `:tcd`, `:lcd`, `:chdir`, `:tchdir` and `:lchdir`.
@@ -7320,7 +7323,7 @@ void ex_cd(exarg_T *eap)
     if (vim_chdir(new_dir, scope)) {
       EMSG(_(e_failed));
     } else {
-      post_chdir(scope);
+      post_chdir(scope, true);
       // Echo the new current directory if the command was typed.
       if (KeyTyped || p_verbose >= 5) {
         ex_pwd(eap);

src/nvim/generators/gen_options.lua

@@ -79,6 +79,7 @@ local get_flags = function(o)
     {'pri_mkrc'},
     {'deny_in_modelines', 'P_NO_ML'},
     {'deny_duplicates', 'P_NODUP'},
+    {'modelineexpr', 'P_MLE'},
   }) do
     local key_name = flag_desc[1]
     local def_name = flag_desc[2] or ('P_' .. key_name:upper())

src/nvim/getchar.c

@@ -1244,6 +1244,13 @@ openscript (
     EMSG(_(e_nesting));
     return;
   }
+
+  // Disallow sourcing a file in the sandbox, the commands would be executed
+  // later, possibly outside of the sandbox.
+  if (check_secure()) {
+    return;
+  }
+
   if (ignore_script)
     /* Not reading from script, also don't open one.  Warning message? */
     return;

src/nvim/memfile.c

@@ -76,7 +76,8 @@
 
 /// @param flags  Flags for open() call.
 ///
-/// @return The open memory file.
+/// @return - The open memory file, on success.
+///         - NULL, on failure (e.g. file does not exist).
 memfile_T *mf_open(char_u *fname, int flags)
 {
   memfile_T *mfp = xmalloc(sizeof(memfile_T));

src/nvim/memline.c

@@ -277,6 +277,9 @@ int ml_open(buf_T *buf)
 
   // Open the memfile.  No swap file is created yet.
   memfile_T *mfp = mf_open(NULL, 0);
+  if (mfp == NULL) {
+    goto error;
+  }
 
   buf->b_ml.ml_mfp = mfp;
   buf->b_ml.ml_flags = ML_EMPTY;
@@ -360,10 +363,12 @@ int ml_open(buf_T *buf)
   return OK;
 
 error:
-  if (hp) {
-    mf_put(mfp, hp, false, false);
+  if (mfp != NULL) {
+    if (hp) {
+      mf_put(mfp, hp, false, false);
+    }
+    mf_close(mfp, true);  // will also xfree(mfp->mf_fname)
   }
-  mf_close(mfp, true);  // will also xfree(mfp->mf_fname)
   buf->b_ml.ml_mfp = NULL;
   return FAIL;
 }
@@ -839,7 +844,7 @@ void ml_recover(void)
                                     mf_open() will consume "fname_used"! */
   mfp = mf_open(fname_used, O_RDONLY);
   fname_used = p;
-  if (mfp->mf_fd < 0) {
+  if (mfp == NULL || mfp->mf_fd < 0) {
     EMSG2(_("E306: Cannot open %s"), fname_used);
     goto theend;
   }

src/nvim/option.c

@@ -251,6 +251,7 @@ typedef struct vimoption {
 #define P_RWINONLY     0x10000000U  ///< only redraw current window
 #define P_NDNAME       0x20000000U  ///< only normal dir name chars allowed
 #define P_UI_OPTION    0x40000000U  ///< send option to remote ui
+#define P_MLE          0x80000000U  ///< under control of 'modelineexpr'
 
 #define HIGHLIGHT_INIT \
   "8:SpecialKey,~:EndOfBuffer,z:TermCursor,Z:TermCursorNC,@:NonText," \
@@ -306,6 +307,15 @@ static char *(p_cot_values[]) =       { "menu", "menuone", "longest", "preview",
 static char *(p_icm_values[]) =       { "nosplit", "split", NULL };
 static char *(p_scl_values[]) =       { "yes", "no", "auto", NULL };
 
+/// All possible flags for 'shm'.
+static char_u SHM_ALL[] = {
+  SHM_RO, SHM_MOD, SHM_FILE, SHM_LAST, SHM_TEXT, SHM_LINES, SHM_NEW, SHM_WRI,
+  SHM_ABBREVIATIONS, SHM_WRITE, SHM_TRUNC, SHM_TRUNCALL, SHM_OVER,
+  SHM_OVERALL, SHM_SEARCH, SHM_ATTENTION, SHM_INTRO, SHM_COMPLETIONMENU,
+  SHM_RECORDING, SHM_FILEINFO,
+  0,
+};
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "option.c.generated.h"
 #endif
@@ -1200,7 +1210,7 @@ do_set (
         }
         len++;
         if (opt_idx == -1) {
-          key = find_key_option(arg + 1);
+          key = find_key_option(arg + 1, true);
         }
       } else {
         len = 0;
@@ -1214,7 +1224,7 @@ do_set (
         }
         opt_idx = findoption_len((const char *)arg, (size_t)len);
         if (opt_idx == -1) {
-          key = find_key_option(arg);
+          key = find_key_option(arg, false);
         }
       }
 
@@ -1281,6 +1291,11 @@ do_set (
           errmsg = (char_u *)_("E520: Not allowed in a modeline");
           goto skip;
         }
+        if ((flags & P_MLE) && !p_mle) {
+          errmsg = (char_u *)_(
+              "E992: Not allowed in a modeline when 'modelineexpr' is off");
+          goto skip;
+        }
         /* In diff mode some options are overruled.  This avoids that
          * 'foldmethod' becomes "marker" instead of "diff" and that
          * "wrap" gets set. */
@@ -1355,6 +1370,10 @@ do_set (
             && nextchar != NUL && !ascii_iswhite(afterchar))
           errmsg = e_trailing;
       } else {
+
+        int value_is_replaced = !prepending && !adding && !removing;
+        int value_checked = false;
+
         if (flags & P_BOOL) {                       /* boolean */
           if (nextchar == '=' || nextchar == ':') {
             errmsg = e_invarg;
@@ -1774,12 +1793,32 @@ do_set (
             // buffer is closed by autocommands.
             saved_newval = (newval != NULL) ? xstrdup((char *)newval) : 0;
 
-            // Handle side effects, and set the global value for
-            // ":set" on local options. Note: when setting 'syntax'
-            // or 'filetype' autocommands may be triggered that can
-            // cause havoc.
-            errmsg = did_set_string_option(opt_idx, (char_u **)varp,
-                new_value_alloced, oldval, errbuf, opt_flags);
+            {
+              uint32_t *p = insecure_flag(opt_idx, opt_flags);
+              const int secure_saved = secure;
+
+              // When an option is set in the sandbox, from a
+              // modeline or in secure mode, then deal with side
+              // effects in secure mode.  Also when the value was
+              // set with the P_INSECURE flag and is not
+              // completely replaced.
+              if ((opt_flags & OPT_MODELINE)
+                  || sandbox != 0
+                  || (!value_is_replaced && (*p & P_INSECURE))) {
+                secure = 1;
+              }
+
+              // Handle side effects, and set the global value
+              // for ":set" on local options. Note: when setting
+              // 'syntax' or 'filetype' autocommands may be
+              // triggered that can cause havoc.
+              errmsg = did_set_string_option(opt_idx, (char_u **)varp,
+                                             new_value_alloced, oldval,
+                                             errbuf, sizeof(errbuf),
+                                             opt_flags, &value_checked);
+
+              secure = secure_saved;
+            }
 
             if (errmsg == NULL) {
               if (!starting) {
@@ -1806,8 +1845,7 @@ do_set (
         }
 
         if (opt_idx >= 0)
-          did_set_option(opt_idx, opt_flags,
-              !prepending && !adding && !removing);
+          did_set_option(opt_idx, opt_flags, value_is_replaced, value_checked);
       }
 
 skip:
@@ -1872,7 +1910,9 @@ static void
 did_set_option (
     int opt_idx,
     int opt_flags,              /* possibly with OPT_MODELINE */
-    int new_value              /* value was replaced completely */
+    int new_value,              /* value was replaced completely */
+    int value_checked           /* value was checked to be safe, no need to
+                                   set P_INSECURE */
 )
 {
   options[opt_idx].flags |= P_WAS_SET;
@@ -1881,20 +1921,22 @@ did_set_option (
    * set the P_INSECURE flag.  Otherwise, if a new value is stored reset the
    * flag. */
   uint32_t *p = insecure_flag(opt_idx, opt_flags);
-  if (secure
-      || sandbox != 0
-      || (opt_flags & OPT_MODELINE))
+  if (!value_checked && (secure
+                         || sandbox != 0
+                         || (opt_flags & OPT_MODELINE))) {
     *p = *p | P_INSECURE;
-  else if (new_value)
+  } else if (new_value) {
     *p = *p & ~P_INSECURE;
+  }
 }
 
-static char_u *illegal_char(char_u *errbuf, int c)
+static char_u *illegal_char(char_u *errbuf, size_t errbuflen, int c)
 {
-  if (errbuf == NULL)
+  if (errbuf == NULL) {
     return (char_u *)"";
-  sprintf((char *)errbuf, _("E539: Illegal character <%s>"),
-      (char *)transchar(c));
+  }
+  vim_snprintf((char *)errbuf, errbuflen, _("E539: Illegal character <%s>"),
+               (char *)transchar(c));
   return errbuf;
 }
 
@@ -1904,10 +1946,12 @@ static char_u *illegal_char(char_u *errbuf, int c)
  */
 static int string_to_key(char_u *arg)
 {
-  if (*arg == '<')
-    return find_key_option(arg + 1);
-  if (*arg == '^')
+  if (*arg == '<') {
+    return find_key_option(arg + 1, true);
+  }
+  if (*arg == '^') {
     return Ctrl_chr(arg[1]);
+  }
   return *arg;
 }
 
@@ -2383,10 +2427,12 @@ static char *set_string_option(const int opt_idx, const char *const value,
   char *const saved_oldval = xstrdup(oldval);
   char *const saved_newval = xstrdup(s);
 
+  int value_checked = false;
   char *const r = (char *)did_set_string_option(
-      opt_idx, (char_u **)varp, (int)true, (char_u *)oldval, NULL, opt_flags);
+      opt_idx, (char_u **)varp, (int)true, (char_u *)oldval,
+      NULL, 0, opt_flags, &value_checked);
   if (r == NULL) {
-    did_set_option(opt_idx, opt_flags, true);
+    did_set_option(opt_idx, opt_flags, true, value_checked);
   }
 
   // call autocommand after handling side effects
@@ -2417,23 +2463,21 @@ static bool valid_filetype(char_u *val)
   return true;
 }
 
-#ifdef _MSC_VER
-// MSVC optimizations are disabled for this function because it
-// incorrectly generates an empty string for SHM_ALL.
-#pragma optimize("", off)
-#endif
 /*
  * Handle string options that need some action to perform when changed.
  * Returns NULL for success, or an error message for an error.
  */
 static char_u *
-did_set_string_option (
-    int opt_idx,                            /* index in options[] table */
-    char_u **varp,                     /* pointer to the option variable */
-    int new_value_alloced,                  /* new value was allocated */
-    char_u *oldval,                    /* previous value of the option */
-    char_u *errbuf,                    /* buffer for errors, or NULL */
-    int opt_flags                          /* OPT_LOCAL and/or OPT_GLOBAL */
+did_set_string_option(
+    int opt_idx,                       // index in options[] table
+    char_u **varp,                     // pointer to the option variable
+    int new_value_alloced,             // new value was allocated
+    char_u *oldval,                    // previous value of the option
+    char_u *errbuf,                    // buffer for errors, or NULL
+    size_t errbuflen,                  // length of errors buffer
+    int opt_flags,                     // OPT_LOCAL and/or OPT_GLOBAL
+    int *value_checked                 // value was checked to be safe, no
+                                       // need to set P_INSECURE
 )
 {
   char_u      *errmsg = NULL;
@@ -2651,8 +2695,20 @@ did_set_string_option (
     if (!valid_filetype(*varp)) {
       errmsg = e_invarg;
     } else {
+      int secure_save = secure;
+
+      // Reset the secure flag, since the value of 'keymap' has
+      // been checked to be safe.
+      secure = 0;
+
       // load or unload key mapping tables
       errmsg = keymap_init();
+
+      secure = secure_save;
+
+      // Since we check the value, there is no need to set P_INSECURE,
+      // even when the value comes from a modeline.
+      *value_checked = true;
     }
 
     if (errmsg == NULL) {
@@ -2738,7 +2794,7 @@ did_set_string_option (
       while (*s && *s != ':') {
         if (vim_strchr((char_u *)COM_ALL, *s) == NULL
             && !ascii_isdigit(*s) && *s != '-') {
-          errmsg = illegal_char(errbuf, *s);
+          errmsg = illegal_char(errbuf, errbuflen, *s);
           break;
         }
         ++s;
@@ -2790,7 +2846,7 @@ did_set_string_option (
     for (s = p_shada; *s; ) {
       /* Check it's a valid character */
       if (vim_strchr((char_u *)"!\"%'/:<@cfhnrs", *s) == NULL) {
-        errmsg = illegal_char(errbuf, *s);
+        errmsg = illegal_char(errbuf, errbuflen, *s);
         break;
       }
       if (*s == 'n') {          /* name is always last one */
@@ -2810,9 +2866,9 @@ did_set_string_option (
 
         if (!ascii_isdigit(*(s - 1))) {
           if (errbuf != NULL) {
-            sprintf((char *)errbuf,
-                _("E526: Missing number after <%s>"),
-                transchar_byte(*(s - 1)));
+            vim_snprintf((char *)errbuf, errbuflen,
+                         _("E526: Missing number after <%s>"),
+                         transchar_byte(*(s - 1)));
             errmsg = errbuf;
           } else
             errmsg = (char_u *)"";
@@ -2990,7 +3046,7 @@ did_set_string_option (
       if (!*s)
         break;
       if (vim_strchr((char_u *)".wbuksid]tU", *s) == NULL) {
-        errmsg = illegal_char(errbuf, *s);
+        errmsg = illegal_char(errbuf, errbuflen, *s);
         break;
       }
       if (*++s != NUL && *s != ',' && *s != ' ') {
@@ -3004,9 +3060,9 @@ did_set_string_option (
           }
         } else {
           if (errbuf != NULL) {
-            sprintf((char *)errbuf,
-                _("E535: Illegal character after <%c>"),
-                *--s);
+            vim_snprintf((char *)errbuf, errbuflen,
+                         _("E535: Illegal character after <%c>"),
+                         *--s);
             errmsg = errbuf;
           } else
             errmsg = (char_u *)"";
@@ -3163,12 +3219,20 @@ did_set_string_option (
       errmsg = e_invarg;
     } else {
       value_changed = STRCMP(oldval, *varp) != 0;
+
+      // Since we check the value, there is no need to set P_INSECURE,
+      // even when the value comes from a modeline.
+      *value_checked = true;
     }
   } else if (gvarp == &p_syn) {
     if (!valid_filetype(*varp)) {
       errmsg = e_invarg;
     } else {
       value_changed = STRCMP(oldval, *varp) != 0;
+
+      // Since we check the value, there is no need to set P_INSECURE,
+      // even when the value comes from a modeline.
+      *value_checked = true;
     }
   } else if (varp == &curwin->w_p_winhl) {
     if (!parse_winhl_opt(curwin)) {
@@ -3194,7 +3258,7 @@ did_set_string_option (
     if (p != NULL) {
       for (s = *varp; *s; ++s)
         if (vim_strchr(p, *s) == NULL) {
-          errmsg = illegal_char(errbuf, *s);
+          errmsg = illegal_char(errbuf, errbuflen, *s);
           break;
         }
     }
@@ -3258,6 +3322,11 @@ did_set_string_option (
       // already set to this value.
       if (!(opt_flags & OPT_MODELINE) || value_changed) {
         static int ft_recursive = 0;
+        int secure_save = secure;
+
+        // Reset the secure flag, since the value of 'filetype' has
+        // been checked to be safe.
+        secure = 0;
 
         ft_recursive++;
         did_filetype = true;
@@ -3270,6 +3339,7 @@ did_set_string_option (
         if (varp != &(curbuf->b_p_ft)) {
           varp = NULL;
         }
+        secure = secure_save;
       }
     }
     if (varp == &(curwin->w_s->b_p_spl)) {
@@ -3287,11 +3357,13 @@ did_set_string_option (
        * '.encoding'.
        */
       for (p = q; *p != NUL; ++p)
-        if (vim_strchr((char_u *)"_.,", *p) != NULL)
+        if (!ASCII_ISALNUM(*p) && *p != '-')
           break;
-      vim_snprintf((char *)fname, sizeof(fname), "spell/%.*s.vim",
-                   (int)(p - q), q);
-      source_runtime(fname, DIP_ALL);
+      if (p > q) {
+        vim_snprintf((char *)fname, sizeof(fname), "spell/%.*s.vim",
+                     (int)(p - q), q);
+        source_runtime(fname, DIP_ALL);
+      }
     }
   }
 
@@ -3311,9 +3383,6 @@ did_set_string_option (
 
   return errmsg;
 }
-#ifdef _MSC_VER
-#pragma optimize("", on)
-#endif
 
 /*
  * Simple int comparison function for use with qsort()
@@ -3553,7 +3622,7 @@ char_u *check_stl_option(char_u *s)
       continue;
     }
     if (vim_strchr(STL_ALL, *s) == NULL) {
-      return illegal_char(errbuf, *s);
+      return illegal_char(errbuf, sizeof(errbuf), *s);
     }
     if (*s == '{') {
       s++;
@@ -4892,19 +4961,20 @@ char *set_option_value(const char *const name, const long number,
   return NULL;
 }
 
-/*
- * Translate a string like "t_xx", "<t_xx>" or "<S-Tab>" to a key number.
- */
-int find_key_option_len(const char_u *arg, size_t len)
+// Translate a string like "t_xx", "<t_xx>" or "<S-Tab>" to a key number.
+// When "has_lt" is true there is a '<' before "*arg_arg".
+// Returns 0 when the key is not recognized.
+int find_key_option_len(const char_u *arg_arg, size_t len, bool has_lt)
 {
-  int key;
+  int key = 0;
   int modifiers;
+  const char_u *arg = arg_arg;
 
   // Don't use get_special_key_code() for t_xx, we don't want it to call
   // add_termcap_entry().
   if (len >= 4 && arg[0] == 't' && arg[1] == '_') {
     key = TERMCAP2KEY(arg[2], arg[3]);
-  } else {
+  } else if (has_lt)  {
     arg--;  // put arg at the '<'
     modifiers = 0;
     key = find_special_key(&arg, len + 1, &modifiers, true, true, false);
@@ -4915,9 +4985,9 @@ int find_key_option_len(const char_u *arg, size_t len)
   return key;
 }
 
-static int find_key_option(const char_u *arg)
+static int find_key_option(const char_u *arg, bool has_lt)
 {
-  return find_key_option_len(arg, STRLEN(arg));
+  return find_key_option_len(arg, STRLEN(arg), has_lt);
 }
 
 /*

src/nvim/option_defs.h

@@ -178,14 +178,6 @@ enum {
   SHM_RO, SHM_MOD, SHM_FILE, SHM_LAST, SHM_TEXT, SHM_LINES, SHM_NEW, SHM_WRI, \
   0, \
 })
-/// All possible flags for 'shm'.
-#define SHM_ALL ((char_u[]) { \
-  SHM_RO, SHM_MOD, SHM_FILE, SHM_LAST, SHM_TEXT, SHM_LINES, SHM_NEW, SHM_WRI, \
-  SHM_ABBREVIATIONS, SHM_WRITE, SHM_TRUNC, SHM_TRUNCALL, SHM_OVER, \
-  SHM_OVERALL, SHM_SEARCH, SHM_ATTENTION, SHM_INTRO, SHM_COMPLETIONMENU, \
-  SHM_RECORDING, SHM_FILEINFO, \
-  0, \
-})
 
 /* characters for p_go: */
 #define GO_ASEL         'a'             /* autoselect */
@@ -503,6 +495,7 @@ EXTERN long p_mmd;              // 'maxmapdepth'
 EXTERN long p_mmp;              // 'maxmempattern'
 EXTERN long p_mis;              // 'menuitems'
 EXTERN char_u   *p_msm;         // 'mkspellmem'
+EXTERN long p_mle;              // 'modelineexpr'
 EXTERN long p_mls;              // 'modelines'
 EXTERN char_u   *p_mouse;       // 'mouse'
 EXTERN char_u   *p_mousem;      // 'mousemodel'

src/nvim/options.lua

@@ -8,6 +8,7 @@
 --    defaults={condition=nil, if_true={vi=224, vim=0}, if_false=nil},
 --    secure=nil, gettext=nil, noglob=nil, normal_fname_chars=nil,
 --    pri_mkrc=nil, deny_in_modelines=nil, normal_dname_chars=nil,
+--    modelineexpr=nil,
 --    expand=nil, nodefault=nil, no_mkrc=nil, vi_def=true, vim=true,
 --    alloced=nil,
 --    save_pv_indir=nil,
@@ -286,6 +287,7 @@ return {
       deny_duplicates=true,
       vi_def=true,
       expand=true,
+      secure=true,
       varname='p_cdpath',
       defaults={if_true={vi=",,"}}
     },
@@ -856,6 +858,7 @@ return {
       type='string', scope={'window'},
       vi_def=true,
       vim=true,
+      modelineexpr=true,
       alloced=true,
       redraw={'current_window'},
       defaults={if_true={vi="0"}}
@@ -931,6 +934,7 @@ return {
       type='string', scope={'window'},
       vi_def=true,
       vim=true,
+      modelineexpr=true,
       alloced=true,
       redraw={'current_window'},
       defaults={if_true={vi="foldtext()"}}
@@ -940,6 +944,7 @@ return {
       type='string', scope={'buffer'},
       vi_def=true,
       vim=true,
+      modelineexpr=true,
       alloced=true,
       varname='p_fex',
       defaults={if_true={vi=""}}
@@ -1053,6 +1058,7 @@ return {
       full_name='guitablabel', abbreviation='gtl',
       type='string', scope={'global'},
       vi_def=true,
+      modelineexpr=true,
       redraw={'current_window'},
       enable_if=false,
     },
@@ -1143,6 +1149,7 @@ return {
       full_name='iconstring',
       type='string', scope={'global'},
       vi_def=true,
+      modelineexpr=true,
       varname='p_iconstring',
       defaults={if_true={vi=""}}
     },
@@ -1209,6 +1216,7 @@ return {
       full_name='includeexpr', abbreviation='inex',
       type='string', scope={'buffer'},
       vi_def=true,
+      modelineexpr=true,
       alloced=true,
       varname='p_inex',
       defaults={if_true={vi=""}}
@@ -1225,6 +1233,7 @@ return {
       type='string', scope={'buffer'},
       vi_def=true,
       vim=true,
+      modelineexpr=true,
       alloced=true,
       varname='p_inde',
       defaults={if_true={vi=""}}
@@ -1538,6 +1547,14 @@ return {
       varname='p_ml',
       defaults={if_true={vi=false, vim=true}}
     },
+    {
+      full_name='modelineexpr', abbreviation='mle',
+      type='bool', scope={'global'},
+      vi_def=true,
+      secure=true,
+      varname='p_mle',
+      defaults={if_true={vi=false}}
+    },
     {
       full_name='modelines', abbreviation='mls',
       type='number', scope={'global'},
@@ -1898,6 +1915,7 @@ return {
       type='string', scope={'global'},
       vi_def=true,
       alloced=true,
+      modelineexpr=true,
       redraw={'statuslines'},
       varname='p_ruf',
       defaults={if_true={vi=""}}
@@ -2293,6 +2311,7 @@ return {
       type='string', scope={'global', 'window'},
       vi_def=true,
       alloced=true,
+      modelineexpr=true,
       redraw={'statuslines'},
       varname='p_stl',
       defaults={if_true={vi=""}}
@@ -2352,6 +2371,7 @@ return {
       full_name='tabline', abbreviation='tal',
       type='string', scope={'global'},
       vi_def=true,
+      modelineexpr=true,
       redraw={'all_windows'},
       varname='p_tal',
       defaults={if_true={vi=""}}
@@ -2511,6 +2531,7 @@ return {
       full_name='titlestring',
       type='string', scope={'global'},
       vi_def=true,
+      modelineexpr=true,
       varname='p_titlestring',
       defaults={if_true={vi=""}}
     },

src/nvim/regexp.c

@@ -4920,7 +4920,7 @@ regmatch (
               }
             } else {
               const char_u *const line =
-                  reg_getline(behind_pos.rs_u.pos.lnum);
+                  reg_getline(rp->rs_un.regsave.rs_u.pos.lnum);
 
               rp->rs_un.regsave.rs_u.pos.col -=
                   utf_head_off(line,

src/nvim/screen.c

@@ -5338,10 +5338,11 @@ void screen_puts_len(char_u *text, int textlen, int row, int col, int attr)
     schar_from_ascii(ScreenLines[off - 1], ' ');
     ScreenAttrs[off - 1] = 0;
     // redraw the previous cell, make it empty
-    if (put_dirty_first == -1) {
+    if (put_dirty_first == -1 || col-1 < put_dirty_first) {
       put_dirty_first = col-1;
     }
     put_dirty_last = col+1;
+    put_dirty_last = MAX(put_dirty_last, col+1);
     // force the cell at "col" to be redrawn
     force_redraw_next = true;
   }
@@ -5422,10 +5423,10 @@ void screen_puts_len(char_u *text, int textlen, int row, int col, int attr)
         ScreenLines[off + 1][0] = 0;
         ScreenAttrs[off + 1] = attr;
       }
-      if (put_dirty_first == -1) {
+      if (put_dirty_first == -1 || col < put_dirty_first) {
         put_dirty_first = col;
       }
-      put_dirty_last = col+mbyte_cells;
+      put_dirty_last = MAX(put_dirty_last, col+mbyte_cells);
     }
 
     off += mbyte_cells;

src/nvim/spell.c

@@ -2510,7 +2510,7 @@ buf_T *open_spellbuf(void)
   buf->b_spell = true;
   buf->b_p_swf = true;        // may create a swap file
   if (ml_open(buf) == FAIL) {
-    abort();
+    ELOG("Error opening a new memline");
   }
   ml_open_file(buf);          // create swap file now
 

src/nvim/terminal.c

@@ -1304,8 +1304,6 @@ static void redraw(bool restore_cursor)
 
 static void adjust_topline(Terminal *term, buf_T *buf, long added)
 {
-  int height, width;
-  vterm_get_size(term->vt, &height, &width);
   FOR_ALL_WINDOWS_IN_TAB(wp, curtab) {
     if (wp->w_buffer == buf) {
       linenr_T ml_end = buf->b_ml.ml_line_count;
@@ -1314,7 +1312,7 @@ static void adjust_topline(Terminal *term, buf_T *buf, long added)
       if (following || (wp == curwin && is_focused(term))) {
         // "Follow" the terminal output
         wp->w_cursor.lnum = ml_end;
-        set_topline(wp, MAX(wp->w_cursor.lnum - height + 1, 1));
+        set_topline(wp, MAX(wp->w_cursor.lnum - wp->w_height + 1, 1));
       } else {
         // Ensure valid cursor for each window displaying this terminal.
         wp->w_cursor.lnum = MIN(wp->w_cursor.lnum, ml_end);

src/nvim/testdir/test49.vim

@@ -1,6 +1,6 @@
 " Vim script language tests
 " Author:	Servatius Brandt <Servatius.Brandt@fujitsu-siemens.com>
-" Last Change:	2016 Feb 07
+" Last Change:	2019 May 24
 
 "-------------------------------------------------------------------------------
 " Test environment							    {{{1
@@ -9005,5 +9005,4 @@ Xcheck 50443995
 "-------------------------------------------------------------------------------
 " Modelines								    {{{1
 " vim: ts=8 sw=4 tw=80 fdm=marker
-" vim: fdt=substitute(substitute(foldtext(),\ '\\%(^+--\\)\\@<=\\(\\s*\\)\\(.\\{-}\\)\:\ \\%(\"\ \\)\\=\\(Test\ \\d*\\)\:\\s*',\ '\\3\ (\\2)\:\ \\1',\ \"\"),\ '\\(Test\\s*\\)\\(\\d\\)\\D\\@=',\ '\\1\ \\2',\ "")
 "-------------------------------------------------------------------------------

src/nvim/testdir/test_alot.vim

@@ -28,6 +28,7 @@ source test_lambda.vim
 source test_mapping.vim
 source test_menu.vim
 source test_messages.vim
+source test_modeline.vim
 source test_move.vim
 source test_partial.vim
 source test_popup.vim

src/nvim/testdir/test_autocmd.vim

@@ -652,6 +652,29 @@ func Test_OptionSet_diffmode_close()
   "delfunc! AutoCommandOptionSet
 endfunc
 
+func Test_OptionSet_modeline()
+  throw 'skipped: Nvim does not support test_override()'
+  call test_override('starting', 1)
+  au! OptionSet
+  augroup set_tabstop
+    au OptionSet tabstop call timer_start(1, {-> execute("echo 'Handler called'", "")})
+  augroup END
+  call writefile(['vim: set ts=7 sw=5 :', 'something'], 'XoptionsetModeline')
+  set modeline
+  let v:errmsg = ''
+  call assert_fails('split XoptionsetModeline', 'E12:')
+  call assert_equal(7, &ts)
+  call assert_equal('', v:errmsg)
+
+  augroup set_tabstop
+    au!
+  augroup END
+  bwipe!
+  set ts&
+  call delete('XoptionsetModeline')
+  call test_override('starting', 0)
+endfunc
+
 " Test for Bufleave autocommand that deletes the buffer we are about to edit.
 func Test_BufleaveWithDelete()
   new | edit Xfile1

src/nvim/testdir/test_functions.vim

@@ -1037,3 +1037,19 @@ func Test_func_range_with_edit()
   call delete('Xfuncrange2')
   bwipe!
 endfunc
+
+sandbox function Fsandbox()
+  normal ix
+endfunc
+
+func Test_func_sandbox()
+  sandbox let F = {-> 'hello'}
+  call assert_equal('hello', F())
+
+  sandbox let F = {-> execute("normal ix\<Esc>")}
+  call assert_fails('call F()', 'E48:')
+  unlet F
+
+  call assert_fails('call Fsandbox()', 'E48:')
+  delfunc Fsandbox
+endfunc

src/nvim/testdir/test_glob2regpat.vim

@@ -1,12 +1,12 @@
 " Test glob2regpat()
 
-func Test_invalid()
+func Test_glob2regpat_invalid()
   call assert_fails('call glob2regpat(1.33)', 'E806:')
   call assert_fails('call glob2regpat("}")', 'E219:')
   call assert_fails('call glob2regpat("{")', 'E220:')
 endfunc
 
-func Test_valid()
+func Test_glob2regpat_valid()
   call assert_equal('^foo\.', glob2regpat('foo.*'))
   call assert_equal('^foo.$', glob2regpat('foo?'))
   call assert_equal('\.vim$', glob2regpat('*.vim'))

src/nvim/testdir/test_modeline.vim

@@ -0,0 +1,173 @@
+" Tests for parsing the modeline.
+
+func Test_modeline_invalid()
+  " This was reading allocated memory in the past.
+  call writefile(['vi:0', 'nothing'], 'Xmodeline')
+  let modeline = &modeline
+  set modeline
+  call assert_fails('set Xmodeline', 'E518:')
+
+  let &modeline = modeline
+  bwipe!
+  call delete('Xmodeline')
+ endfunc
+
+func Test_modeline_filetype()
+  call writefile(['vim: set ft=c :', 'nothing'], 'Xmodeline_filetype')
+  let modeline = &modeline
+  set modeline
+  filetype plugin on
+  split Xmodeline_filetype
+  call assert_equal("c", &filetype)
+  call assert_equal(1, b:did_ftplugin)
+  call assert_equal("ccomplete#Complete", &ofu)
+
+  bwipe!
+  call delete('Xmodeline_filetype')
+  let &modeline = modeline
+  filetype plugin off
+endfunc
+
+func Test_modeline_syntax()
+  call writefile(['vim: set syn=c :', 'nothing'], 'Xmodeline_syntax')
+  let modeline = &modeline
+  set modeline
+  syntax enable
+  split Xmodeline_syntax
+  call assert_equal("c", &syntax)
+  call assert_equal("c", b:current_syntax)
+
+  bwipe!
+  call delete('Xmodeline_syntax')
+  let &modeline = modeline
+  syntax off
+endfunc
+
+func Test_modeline_keymap()
+  if !has('keymap')
+    return
+  endif
+  call writefile(['vim: set keymap=greek :', 'nothing'], 'Xmodeline_keymap')
+  let modeline = &modeline
+  set modeline
+  split Xmodeline_keymap
+  call assert_equal("greek", &keymap)
+  call assert_match('greek\|grk', b:keymap_name)
+
+  bwipe!
+  call delete('Xmodeline_keymap')
+  let &modeline = modeline
+  set keymap= iminsert=0 imsearch=-1
+endfunc
+
+func s:modeline_fails(what, text, error)
+  if !exists('+' . a:what)
+    return
+  endif
+  let fname = "Xmodeline_fails_" . a:what
+  call writefile(['vim: set ' . a:text . ' :', 'nothing'], fname)
+  let modeline = &modeline
+  set modeline
+  filetype plugin on
+  syntax enable
+  call assert_fails('split ' . fname, a:error)
+  call assert_equal("", &filetype)
+  call assert_equal("", &syntax)
+
+  bwipe!
+  call delete(fname)
+  let &modeline = modeline
+  filetype plugin off
+  syntax off
+endfunc
+
+func Test_modeline_filetype_fails()
+  call s:modeline_fails('filetype', 'ft=evil$CMD', 'E474:')
+endfunc
+
+func Test_modeline_syntax_fails()
+  call s:modeline_fails('syntax', 'syn=evil$CMD', 'E474:')
+endfunc
+
+func Test_modeline_keymap_fails()
+  call s:modeline_fails('keymap', 'keymap=evil$CMD', 'E474:')
+endfunc
+
+func Test_modeline_fails_always()
+  call s:modeline_fails('backupdir', 'backupdir=Something()', 'E520:')
+  call s:modeline_fails('cdpath', 'cdpath=Something()', 'E520:')
+  call s:modeline_fails('charconvert', 'charconvert=Something()', 'E520:')
+  call s:modeline_fails('completefunc', 'completefunc=Something()', 'E520:')
+  call s:modeline_fails('cscopeprg', 'cscopeprg=Something()', 'E520:')
+  call s:modeline_fails('diffexpr', 'diffexpr=Something()', 'E520:')
+  call s:modeline_fails('directory', 'directory=Something()', 'E520:')
+  call s:modeline_fails('equalprg', 'equalprg=Something()', 'E520:')
+  call s:modeline_fails('errorfile', 'errorfile=Something()', 'E520:')
+  call s:modeline_fails('exrc', 'exrc=Something()', 'E520:')
+  call s:modeline_fails('formatprg', 'formatprg=Something()', 'E520:')
+  call s:modeline_fails('fsync', 'fsync=Something()', 'E520:')
+  call s:modeline_fails('grepprg', 'grepprg=Something()', 'E520:')
+  call s:modeline_fails('helpfile', 'helpfile=Something()', 'E520:')
+  call s:modeline_fails('imactivatefunc', 'imactivatefunc=Something()', 'E520:')
+  call s:modeline_fails('imstatusfunc', 'imstatusfunc=Something()', 'E520:')
+  call s:modeline_fails('imstyle', 'imstyle=Something()', 'E520:')
+  call s:modeline_fails('keywordprg', 'keywordprg=Something()', 'E520:')
+  call s:modeline_fails('langmap', 'langmap=Something()', 'E520:')
+  call s:modeline_fails('luadll', 'luadll=Something()', 'E520:')
+  call s:modeline_fails('makeef', 'makeef=Something()', 'E520:')
+  call s:modeline_fails('makeprg', 'makeprg=Something()', 'E520:')
+  call s:modeline_fails('mkspellmem', 'mkspellmem=Something()', 'E520:')
+  call s:modeline_fails('mzschemedll', 'mzschemedll=Something()', 'E520:')
+  call s:modeline_fails('mzschemegcdll', 'mzschemegcdll=Something()', 'E520:')
+  call s:modeline_fails('modelineexpr', 'modelineexpr=Something()', 'E520:')
+  call s:modeline_fails('omnifunc', 'omnifunc=Something()', 'E520:')
+  call s:modeline_fails('operatorfunc', 'operatorfunc=Something()', 'E520:')
+  call s:modeline_fails('perldll', 'perldll=Something()', 'E520:')
+  call s:modeline_fails('printdevice', 'printdevice=Something()', 'E520:')
+  call s:modeline_fails('patchexpr', 'patchexpr=Something()', 'E520:')
+  call s:modeline_fails('printexpr', 'printexpr=Something()', 'E520:')
+  call s:modeline_fails('pythondll', 'pythondll=Something()', 'E520:')
+  call s:modeline_fails('pythonhome', 'pythonhome=Something()', 'E520:')
+  call s:modeline_fails('pythonthreedll', 'pythonthreedll=Something()', 'E520:')
+  call s:modeline_fails('pythonthreehome', 'pythonthreehome=Something()', 'E520:')
+  call s:modeline_fails('pyxversion', 'pyxversion=Something()', 'E520:')
+  call s:modeline_fails('rubydll', 'rubydll=Something()', 'E520:')
+  call s:modeline_fails('runtimepath', 'runtimepath=Something()', 'E520:')
+  call s:modeline_fails('secure', 'secure=Something()', 'E520:')
+  call s:modeline_fails('shell', 'shell=Something()', 'E520:')
+  call s:modeline_fails('shellcmdflag', 'shellcmdflag=Something()', 'E520:')
+  call s:modeline_fails('shellpipe', 'shellpipe=Something()', 'E520:')
+  call s:modeline_fails('shellquote', 'shellquote=Something()', 'E520:')
+  call s:modeline_fails('shellredir', 'shellredir=Something()', 'E520:')
+  call s:modeline_fails('shellxquote', 'shellxquote=Something()', 'E520:')
+  call s:modeline_fails('spellfile', 'spellfile=Something()', 'E520:')
+  call s:modeline_fails('spellsuggest', 'spellsuggest=Something()', 'E520:')
+  call s:modeline_fails('tcldll', 'tcldll=Something()', 'E520:')
+  call s:modeline_fails('titleold', 'titleold=Something()', 'E520:')
+  call s:modeline_fails('viewdir', 'viewdir=Something()', 'E520:')
+  call s:modeline_fails('viminfo', 'viminfo=Something()', 'E520:')
+  call s:modeline_fails('viminfofile', 'viminfofile=Something()', 'E520:')
+  call s:modeline_fails('winptydll', 'winptydll=Something()', 'E520:')
+  call s:modeline_fails('undodir', 'undodir=Something()', 'E520:')
+  " only check a few terminal options
+  " Skip these since nvim doesn't support termcodes as options
+  "call s:modeline_fails('t_AB', 't_AB=Something()', 'E520:')
+  "call s:modeline_fails('t_ce', 't_ce=Something()', 'E520:')
+  "call s:modeline_fails('t_sr', 't_sr=Something()', 'E520:')
+  "call s:modeline_fails('t_8b', 't_8b=Something()', 'E520:')
+endfunc
+
+func Test_modeline_fails_modelineexpr()
+  call s:modeline_fails('balloonexpr', 'balloonexpr=Something()', 'E992:')
+  call s:modeline_fails('foldexpr', 'foldexpr=Something()', 'E992:')
+  call s:modeline_fails('foldtext', 'foldtext=Something()', 'E992:')
+  call s:modeline_fails('formatexpr', 'formatexpr=Something()', 'E992:')
+  call s:modeline_fails('guitablabel', 'guitablabel=Something()', 'E992:')
+  call s:modeline_fails('iconstring', 'iconstring=Something()', 'E992:')
+  call s:modeline_fails('includeexpr', 'includeexpr=Something()', 'E992:')
+  call s:modeline_fails('indentexpr', 'indentexpr=Something()', 'E992:')
+  call s:modeline_fails('rulerformat', 'rulerformat=Something()', 'E992:')
+  call s:modeline_fails('statusline', 'statusline=Something()', 'E992:')
+  call s:modeline_fails('tabline', 'tabline=Something()', 'E992:')
+  call s:modeline_fails('titlestring', 'titlestring=Something()', 'E992:')
+endfunc

src/nvim/testdir/test_vimscript.vim

@@ -1297,5 +1297,4 @@ endfunc
 "-------------------------------------------------------------------------------
 " Modelines								    {{{1
 " vim: ts=8 sw=4 tw=80 fdm=marker
-" vim: fdt=substitute(substitute(foldtext(),\ '\\%(^+--\\)\\@<=\\(\\s*\\)\\(.\\{-}\\)\:\ \\%(\"\ \\)\\=\\(Test\ \\d*\\)\:\\s*',\ '\\3\ (\\2)\:\ \\1',\ \"\"),\ '\\(Test\\s*\\)\\(\\d\\)\\D\\@=',\ '\\1\ \\2',\ "")
 "-------------------------------------------------------------------------------

src/nvim/tui/tui.c

@@ -1675,8 +1675,7 @@ static void patch_terminfo_bugs(TUIData *data, const char *term,
   }
 
   // Blacklist of terminals that cannot be trusted to report DECSCUSR support.
-  if (!(st || (vte_version != 0 && vte_version < 3900)
-        || (konsolev > 0 && konsolev < 180770))) {
+  if (!(st || (vte_version != 0 && vte_version < 3900) || konsolev)) {
     data->unibi_ext.reset_cursor_style = unibi_find_ext_str(ut, "Se");
     data->unibi_ext.set_cursor_style = unibi_find_ext_str(ut, "Ss");
   }

test/functional/eval/api_functions_spec.lua

@@ -4,7 +4,8 @@ local lfs = require('lfs')
 local neq, eq, command = helpers.neq, helpers.eq, helpers.command
 local clear, curbufmeths = helpers.clear, helpers.curbufmeths
 local exc_exec, expect, eval = helpers.exc_exec, helpers.expect, helpers.eval
-local insert = helpers.insert
+local insert, meth_pcall = helpers.insert, helpers.meth_pcall
+local meths = helpers.meths
 
 describe('api functions', function()
   before_each(clear)
@@ -145,4 +146,10 @@ describe('api functions', function()
     ]])
     screen:detach()
   end)
+
+  it('cannot be called from sandbox', function()
+    eq({false, 'Vim(call):E48: Not allowed in sandbox'},
+       meth_pcall(command, "sandbox call nvim_input('ievil')"))
+    eq({''}, meths.buf_get_lines(0, 0, -1, true))
+  end)
 end)

test/functional/ex_cmds/cd_spec.lua

@@ -286,6 +286,15 @@ describe("getcwd()", function ()
     command("call delete('../"..directories.global.."', 'd')")
     eq("", helpers.eval("getcwd()"))
   end)
+
+  it("works with 'autochdir' after local directory was set (#9892)", function()
+    local curdir = cwd()
+    command('lcd ' .. directories.global)
+    command('lcd -')
+    command('set autochdir')
+    command('edit ' .. directories.global .. '/foo')
+    eq(curdir .. pathsep .. directories.global, cwd())
+  end)
 end)
 
 

test/functional/ex_cmds/swapfile_preserve_recover_spec.lua

@@ -1,10 +1,11 @@
 local Screen = require('test.functional.ui.screen')
 local helpers = require('test.functional.helpers')(after_each)
 local lfs = require('lfs')
-local feed_command, eq, eval, expect, source =
-  helpers.feed_command, helpers.eq, helpers.eval, helpers.expect, helpers.source
+local eq, eval, expect, source =
+  helpers.eq, helpers.eval, helpers.expect, helpers.source
 local clear = helpers.clear
 local command = helpers.command
+local expect_err = helpers.expect_err
 local feed = helpers.feed
 local nvim_prog = helpers.nvim_prog
 local ok = helpers.ok
@@ -17,9 +18,14 @@ describe(':recover', function()
   before_each(clear)
 
   it('fails if given a non-existent swapfile', function()
-    local swapname = 'bogus-swapfile'
-    feed_command('recover '..swapname) -- This should not segfault. #2117
-    eq('E305: No swap file found for '..swapname, eval('v:errmsg'))
+    local swapname = 'bogus_swapfile'
+    local swapname2 = 'bogus_swapfile.swp'
+    expect_err('E305: No swap file found for '..swapname,
+               command, 'recover '..swapname)  -- Should not segfault. #2117
+    -- Also check filename ending with ".swp". #9504
+    expect_err('Vim%(recover%):E306: Cannot open '..swapname2,
+               command, 'recover '..swapname2)  -- Should not segfault. #2117
+    eq(2, eval('1+1'))  -- Still alive?
   end)
 
 end)

test/functional/provider/python3_spec.lua

@@ -11,8 +11,9 @@ do
   clear()
   if missing_provider('python3') then
     it(':python3 reports E319 if provider is missing', function()
-      expect_err([[Vim%(python3%):E319: No "python3" provider found.*]],
-                 command, 'python3 print("foo")')
+      local expected = [[Vim%(py3.*%):E319: No "python3" provider found.*]]
+      expect_err(expected, command, 'py3 print("foo")')
+      expect_err(expected, command, 'py3file foo')
     end)
     pending('Python 3 (or the pynvim module) is broken/missing', function() end)
     return

test/functional/provider/python_spec.lua

@@ -19,8 +19,9 @@ do
   clear()
   if missing_provider('python') then
     it(':python reports E319 if provider is missing', function()
-      expect_err([[Vim%(python%):E319: No "python" provider found.*]],
-                 command, 'python print("foo")')
+      local expected = [[Vim%(py.*%):E319: No "python" provider found.*]]
+      expect_err(expected, command, 'py print("foo")')
+      expect_err(expected, command, 'pyfile foo')
     end)
     pending('Python 2 (or the pynvim module) is broken/missing', function() end)
     return

test/functional/provider/ruby_spec.lua

@@ -19,8 +19,9 @@ do
   clear()
   if missing_provider('ruby') then
     it(':ruby reports E319 if provider is missing', function()
-      expect_err([[Vim%(ruby%):E319: No "ruby" provider found.*]],
-      command, 'ruby puts "foo"')
+      local expected = [[Vim%(ruby.*%):E319: No "ruby" provider found.*]]
+      expect_err(expected, command, 'ruby puts "foo"')
+      expect_err(expected, command, 'rubyfile foo')
     end)
     pending("Missing neovim RubyGem.", function() end)
     return

test/functional/terminal/altscreen_spec.lua

@@ -8,7 +8,7 @@ local exit_altscreen = thelpers.exit_altscreen
 
 if helpers.pending_win32(pending) then return end
 
-describe('terminal altscreen', function()
+describe(':terminal altscreen', function()
   local screen
 
   before_each(function()

test/functional/terminal/buffer_spec.lua

@@ -6,7 +6,7 @@ local eval, feed_command, source = helpers.eval, helpers.feed_command, helpers.s
 local eq, neq = helpers.eq, helpers.neq
 local write_file = helpers.write_file
 
-describe('terminal buffer', function()
+describe(':terminal buffer', function()
   local screen
 
   before_each(function()

test/functional/terminal/cursor_spec.lua

@@ -7,7 +7,7 @@ local feed_command = helpers.feed_command
 local hide_cursor = thelpers.hide_cursor
 local show_cursor = thelpers.show_cursor
 
-describe('terminal cursor', function()
+describe(':terminal cursor', function()
   local screen
 
   before_each(function()

test/functional/terminal/highlight_spec.lua

@@ -5,7 +5,7 @@ local feed, clear, nvim = helpers.feed, helpers.clear, helpers.nvim
 local nvim_dir, command = helpers.nvim_dir, helpers.command
 local eq, eval = helpers.eq, helpers.eval
 
-describe('terminal window highlighting', function()
+describe(':terminal window highlighting', function()
   local screen
 
   before_each(function()

test/functional/terminal/mouse_spec.lua

@@ -4,7 +4,7 @@ local clear, eq, eval = helpers.clear, helpers.eq, helpers.eval
 local feed, nvim = helpers.feed, helpers.nvim
 local feed_data = thelpers.feed_data
 
-describe('terminal mouse', function()
+describe(':terminal mouse', function()
   local screen
 
   before_each(function()

test/functional/terminal/scrollback_spec.lua

@@ -12,7 +12,7 @@ local curbufmeths = helpers.curbufmeths
 local nvim = helpers.nvim
 local feed_data = thelpers.feed_data
 
-describe('terminal scrollback', function()
+describe(':terminal scrollback', function()
   local screen
 
   before_each(function()
@@ -344,7 +344,7 @@ describe('terminal scrollback', function()
   end)
 end)
 
-describe('terminal prints more lines than the screen height and exits', function()
+describe(':terminal prints more lines than the screen height and exits', function()
   it('will push extra lines to scrollback', function()
     clear()
     local screen = Screen.new(30, 7)
@@ -460,7 +460,7 @@ describe("'scrollback' option", function()
     screen:detach()
   end)
 
-  it('defaults to 10000 in terminal buffers', function()
+  it('defaults to 10000 in :terminal buffers', function()
     set_fake_shell()
     command('terminal')
     eq(10000, curbufmeths.get_option('scrollback'))

test/functional/terminal/tui_spec.lua

@@ -20,7 +20,7 @@ local read_file = helpers.read_file
 
 if helpers.pending_win32(pending) then return end
 
-describe('tui', function()
+describe('TUI', function()
   local screen
 
   before_each(function()

test/functional/terminal/window_spec.lua

@@ -3,8 +3,12 @@ local thelpers = require('test.functional.terminal.helpers')
 local feed, clear = helpers.feed, helpers.clear
 local wait = helpers.wait
 local iswin = helpers.iswin
+local command = helpers.command
+local retry = helpers.retry
+local eq = helpers.eq
+local eval = helpers.eval
 
-describe('terminal window', function()
+describe(':terminal window', function()
   local screen
 
   before_each(function()
@@ -12,6 +16,19 @@ describe('terminal window', function()
     screen = thelpers.screen_setup()
   end)
 
+  it('sets topline correctly #8556', function()
+    -- Test has hardcoded assumptions of dimensions.
+    eq(7, eval('&lines'))
+    command('set shell=sh')
+    command('terminal')
+    retry(nil, nil, function() assert(nil ~= eval('b:terminal_job_pid')) end)
+    -- Terminal/shell contents must exceed the height of this window.
+    command('topleft 1split')
+    feed([[i<cr>]])
+    -- Check topline _while_ in terminal-mode.
+    retry(nil, nil, function() eq(6, eval('winsaveview()["topline"]')) end)
+  end)
+
   describe("with 'number'", function()
     it('wraps text', function()
       feed([[<C-\><C-N>]])

test/functional/terminal/window_split_tab_spec.lua

@@ -9,7 +9,7 @@ local eval = helpers.eval
 local iswin = helpers.iswin
 local retry = helpers.retry
 
-describe('terminal', function()
+describe(':terminal', function()
   local screen
 
   before_each(function()

test/functional/ui/inccommand_spec.lua

@@ -2512,6 +2512,9 @@ describe(":substitute", function()
 end)
 
 it(':substitute with inccommand during :terminal activity', function()
+  if helpers.skip_fragile(pending) then
+    return
+  end
   retry(2, 40000, function()
     local screen = Screen.new(30,15)
     clear()

test/functional/ui/input_spec.lua

@@ -61,6 +61,9 @@ describe('feeding large chunks of input with <Paste>', function()
   end)
 
   it('ok', function()
+    if helpers.skip_fragile(pending) then
+      return
+    end
     local t = {}
     for i = 1, 20000 do
       t[i] = 'item ' .. tostring(i)

test/functional/ui/output_spec.lua

@@ -51,6 +51,9 @@ describe("shell command :!", function()
   end)
 
   it("throttles shell-command output greater than ~10KB", function()
+    if helpers.skip_fragile(pending) then
+      return
+    end
     child_session.feed_data(
       ":!for i in $(seq 2 30000); do echo XXXXXXXXXX $i; done\n")
 

third-party/CMakeLists.txt

@@ -163,8 +163,8 @@ set(GPERF_SHA256 588546b945bba4b70b6a3a616e80b4ab466e3f33024a352fc2198112cdbb3ae
 set(WINTOOLS_URL https://github.com/neovim/deps/raw/2f9acbecf06365c10baa3c0087f34a54c9c6f949/opt/win32tools.zip)
 set(WINTOOLS_SHA256 8bfce7e3a365721a027ce842f2ec1cf878f1726233c215c05964aac07300798c)
 
-set(WINGUI_URL https://github.com/equalsraf/neovim-qt/releases/download/v0.2.11/neovim-qt.zip)
-set(WINGUI_SHA256 b7add4b14561a2d8f55d3bbffb1d887209e4f26f837836639c2efeaaed9fa04e)
+set(WINGUI_URL https://github.com/equalsraf/neovim-qt/releases/download/v0.2.12/neovim-qt.zip)
+set(WINGUI_SHA256 68f8d7d46e21c94ac24577bc65d378c5f622794d11d34950e41edc1b0335560c)
 
 set(WIN32YANK_X86_URL https://github.com/equalsraf/win32yank/releases/download/v0.0.4/win32yank-x86.zip)
 set(WIN32YANK_X86_SHA256 62f34e5a46c5d4a7b3f3b512e1ff7b77fedd432f42581cbe825233a996eed62c)