fix(lua): don't override script ID from :source (#32626)

Problem:  When setting an option, mapping etc. from Lua without -V1, the
          script ID is set to SID_LUA even if there already is a script
          ID assigned by :source.
Solution: Don't set script ID to SID_LUA if it is already a Lua script.
          Also add _editor.lua to ignorelist to make script context more
          useful when using vim.cmd().

src/nvim/api/private/helpers.c

@@ -30,6 +30,7 @@
 #include "nvim/message.h"
 #include "nvim/msgpack_rpc/unpacker.h"
 #include "nvim/pos_defs.h"
+#include "nvim/runtime.h"
 #include "nvim/types_defs.h"
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS
@@ -1057,10 +1058,18 @@ const char *get_default_stl_hl(win_T *wp, bool use_winbar, int stc_hl_id)
 sctx_T api_set_sctx(uint64_t channel_id)
 {
   sctx_T old_current_sctx = current_sctx;
+  // The script context is already properly set when calling an API from Vimscript.
   if (channel_id != VIML_INTERNAL_CALL) {
-    current_sctx.sc_sid = channel_id == LUA_INTERNAL_CALL ? SID_LUA : SID_API_CLIENT;
     current_sctx.sc_lnum = 0;
-    current_sctx.sc_chan = channel_id;
+    if (channel_id == LUA_INTERNAL_CALL) {
+      // When the current script is a Lua script, don't override sc_sid.
+      if (!script_is_lua(current_sctx.sc_sid)) {
+        current_sctx.sc_sid = SID_LUA;
+      }
+    } else {
+      current_sctx.sc_sid = SID_API_CLIENT;
+      current_sctx.sc_chan = channel_id;
+    }
   }
   return old_current_sctx;
 }

src/nvim/eval.c

@@ -7646,22 +7646,8 @@ hashtab_T *find_var_ht_dict(const char *name, const size_t name_len, const char
                  || current_sctx.sc_sid == SID_LUA)
              && current_sctx.sc_sid <= script_items.ga_len) {
     // For anonymous scripts without a script item, create one now so script vars can be used
-    if (current_sctx.sc_sid == SID_LUA) {
+    // Try to resolve lua filename & linenr so it can be shown in last-set messages.
-      // try to resolve lua filename & line no so it can be shown in lastset messages.
+    nlua_set_sctx(&current_sctx);
-      nlua_set_sctx(&current_sctx);
-      if (current_sctx.sc_sid != SID_LUA) {
-        // Great we have valid location. Now here this out we'll create a new
-        // script context with the name and lineno of this one. why ?
-        // for behavioral consistency. With this different anonymous exec from
-        // same file can't access each others script local stuff. We need to do
-        // this all other cases except this will act like that otherwise.
-        bool should_free;
-        // should_free is ignored as script_ctx will be resolved to a fname
-        // and new_script_item() will consume it.
-        char *sc_name = get_scriptname(current_sctx, &should_free);
-        new_script_item(sc_name, &current_sctx.sc_sid);
-      }
-    }
     if (current_sctx.sc_sid == SID_STR || current_sctx.sc_sid == SID_LUA) {
       // Create SID if s: scope is accessed from Lua or anon Vimscript. #15994
       new_script_item(NULL, &current_sctx.sc_sid);

src/nvim/lua/executor.c

@@ -2110,7 +2110,8 @@ bool nlua_execute_on_key(int c, char *typed_buf)
 // @param[out] current
 void nlua_set_sctx(sctx_T *current)
 {
-  if (p_verbose <= 0 || current->sc_sid != SID_LUA) {
+  if (p_verbose <= 0 || (current->sc_sid > 0 && current->sc_lnum > 0)
+      || !script_is_lua(current->sc_sid)) {
     return;
   }
   lua_State *const lstate = global_lstate;
@@ -2119,6 +2120,7 @@ void nlua_set_sctx(sctx_T *current)
   // Files where internal wrappers are defined so we can ignore them
   // like vim.o/opt etc are defined in _options.lua
   char *ignorelist[] = {
+    "vim/_editor.lua",
     "vim/_options.lua",
     "vim/keymap.lua",
   };
@@ -2153,7 +2155,8 @@ void nlua_set_sctx(sctx_T *current)
   if (sid > 0) {
     xfree(source_path);
   } else {
-    new_script_item(source_path, &sid);
+    scriptitem_T *si = new_script_item(source_path, &sid);
+    si->sn_lua = true;
   }
   current->sc_sid = sid;
   current->sc_seq = -1;

src/nvim/runtime.c

@@ -2023,7 +2023,7 @@ static int source_using_linegetter(void *cookie, LineGetter fgetline, const char
   estack_push(ETYPE_SCRIPT, sname, 0);
 
   const sctx_T save_current_sctx = current_sctx;
-  if (current_sctx.sc_sid != SID_LUA) {
+  if (!script_is_lua(current_sctx.sc_sid)) {
     current_sctx.sc_sid = SID_STR;
   }
   current_sctx.sc_seq = 0;
@@ -2241,6 +2241,7 @@ int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
   } else {
     // It's new, generate a new SID.
     si = new_script_item(fname_exp, &sid);
+    si->sn_lua = path_with_extension(fname_exp, "lua");
     fname_exp = xstrdup(si->sn_name);  // used for autocmd
     if (ret_sid != NULL) {
       *ret_sid = sid;
@@ -2268,9 +2269,8 @@ int do_source(char *fname, int check_other, int is_vimrc, int *ret_sid)
 
   cookie.conv.vc_type = CONV_NONE;              // no conversion
 
-  if (path_with_extension(fname_exp, "lua")) {
+  if (si->sn_lua) {
     const sctx_T current_sctx_backup = current_sctx;
-    current_sctx.sc_sid = SID_LUA;
     current_sctx.sc_lnum = 0;
     // Source the file as lua
     nlua_exec_file(fname_exp);
@@ -2356,6 +2356,18 @@ theend:
   return retval;
 }
 
+/// Checks if the script with the given script ID is a Lua script.
+bool script_is_lua(scid_T sid)
+{
+  if (sid == SID_LUA) {
+    return true;
+  }
+  if (!SCRIPT_ID_VALID(sid)) {
+    return false;
+  }
+  return SCRIPT_ITEM(sid)->sn_lua;
+}
+
 /// Find an already loaded script "name".
 /// If found returns its script ID.  If not found returns -1.
 int find_script_by_name(char *name)
@@ -2447,7 +2459,8 @@ char *get_scriptname(sctx_T script_ctx, bool *should_free)
   case SID_STR:
     return _("anonymous :source");
   default: {
-    char *const sname = SCRIPT_ITEM(script_ctx.sc_sid)->sn_name;
+    scriptitem_T *const si = SCRIPT_ITEM(script_ctx.sc_sid);
+    char *sname = si->sn_name;
     if (sname == NULL) {
       snprintf(IObuff, IOSIZE, _("anonymous :source (script id %d)"),
                script_ctx.sc_sid);
@@ -2455,7 +2468,13 @@ char *get_scriptname(sctx_T script_ctx, bool *should_free)
     }
 
     *should_free = true;
-    return home_replace_save(NULL, sname);
+    sname = home_replace_save(NULL, sname);
+    if (si->sn_lua && script_ctx.sc_lnum == 0) {
+      char *const ret = concat_str(sname, _(" (run Nvim with -V1 for more details)"));
+      xfree(sname);
+      return ret;
+    }
+    return sname;
   }
   }
 }

src/nvim/runtime_defs.h

@@ -49,6 +49,7 @@ typedef struct {
   scriptvar_T *sn_vars;         ///< stores s: variables for this script
 
   char *sn_name;
+  bool sn_lua;                  ///< true for a lua script
   bool sn_prof_on;              ///< true when script is/was profiled
   bool sn_pr_force;             ///< forceit: profile functions in this script
   proftime_T sn_pr_child;       ///< time set when going into first child

test/functional/ex_cmds/source_spec.lua

@@ -6,6 +6,7 @@ local insert = n.insert
 local eq = t.eq
 local clear = n.clear
 local api = n.api
+local fn = n.fn
 local feed = n.feed
 local feed_command = n.feed_command
 local write_file = t.write_file
@@ -169,8 +170,9 @@ describe(':source', function()
     eq('4', exec_capture('echo luaeval("y")'))
   end)
 
-  it('can source lua files', function()
+  --- @param verbose boolean
-    local test_file = 'test.lua'
+  local function test_source_lua_file(verbose)
+    local test_file = 'Xtest.lua'
     write_file(
       test_file,
       [[
@@ -178,17 +180,32 @@ describe(':source', function()
       vim.g.sfile_value = vim.fn.expand('<sfile>')
       vim.g.stack_value = vim.fn.expand('<stack>')
       vim.g.script_value = vim.fn.expand('<script>')
+      vim.g.script_id = tonumber(vim.fn.expand('<SID>'):match('<SNR>(%d+)_'))
+      vim.o.mouse = 'nv'
     ]]
     )
 
     command('set shellslash')
-    command('source ' .. test_file)
+    command(('%ssource %s'):format(verbose and 'verbose ' or '', test_file))
     eq(1, eval('g:sourced_lua'))
-    matches([[/test%.lua$]], api.nvim_get_var('sfile_value'))
+    matches([[/Xtest%.lua$]], api.nvim_get_var('sfile_value'))
-    matches([[/test%.lua$]], api.nvim_get_var('stack_value'))
+    matches([[/Xtest%.lua$]], api.nvim_get_var('stack_value'))
-    matches([[/test%.lua$]], api.nvim_get_var('script_value'))
+    matches([[/Xtest%.lua$]], api.nvim_get_var('script_value'))
+
+    local expected_sid = fn.getscriptinfo({ name = test_file })[1].sid
+    local sid = api.nvim_get_var('script_id')
+    eq(expected_sid, sid)
+    eq(sid, api.nvim_get_option_info2('mouse', {}).last_set_sid)
 
     os.remove(test_file)
+  end
+
+  it('can source lua files', function()
+    test_source_lua_file(false)
+  end)
+
+  it('with :verbose modifier can source lua files', function()
+    test_source_lua_file(true)
   end)
 
   describe('can source current buffer', function()
@@ -253,7 +270,7 @@ describe(':source', function()
   end)
 
   it("doesn't throw E484 for lua parsing/runtime errors", function()
-    local test_file = 'test.lua'
+    local test_file = 'Xtest.lua'
 
     -- Does throw E484 for unreadable files
     local ok, result = pcall(exec_capture, ':source ' .. test_file .. 'noexisting')

test/functional/ex_cmds/verbose_spec.lua

@@ -25,6 +25,9 @@ local function last_set_lua_tests(cmd)
 vim.api.nvim_set_option_value('hlsearch', false, {})
 vim.bo.expandtab = true
 vim.opt.number = true
+vim.api.nvim_exec2('set numberwidth=2', {})
+vim.cmd('set colorcolumn=+1')
+
 vim.api.nvim_set_keymap('n', '<leader>key1', ':echo "test"<cr>', {noremap = true})
 vim.keymap.set('n', '<leader>key2', ':echo "test"<cr>')
 
@@ -63,48 +66,34 @@ let &tw = s:return80()\
     exec(cmd .. ' ' .. script_file)
   end)
 
+  local option_checks = {
+    { 'nvim_set_option_value', 'hlsearch', 'nohlsearch' },
+    { 'vim.bo', 'expandtab', '  expandtab' },
+    { 'vim.opt', 'number', '  number' },
+    { 'nvim_exec2', 'numberwidth', '  numberwidth=2' },
+    { 'vim.cmd', 'colorcolumn', '  colorcolumn=+1' },
+  }
+
   teardown(function()
     os.remove(script_file)
   end)
 
-  it('"Last set" for option set by nvim_set_option_value', function()
+  for linenr, check in ipairs(option_checks) do
-    local result = exec_capture(':verbose set hlsearch?')
+    it(('"Last set" for option set by %s'):format(check[1]), function()
-    eq(
+      local result = exec_capture((':verbose set %s?'):format(check[2]))
-      string.format(
+      eq(
-        [[
+        string.format(
-nohlsearch
+          [[
-	Last set from %s line 1]],
+%s
-        script_location
+	Last set from %s line %d]],
-      ),
+          check[3],
-      result
+          script_location,
-    )
+          linenr
-  end)
+        ),
-
+        result
-  it('"Last set" for option set by vim.o', function()
+      )
-    local result = exec_capture(':verbose set expandtab?')
+    end)
-    eq(
+  end
-      string.format(
-        [[
-  expandtab
-	Last set from %s line 2]],
-        script_location
-      ),
-      result
-    )
-  end)
-
-  it('"Last set" for option set by vim.opt', function()
-    local result = exec_capture(':verbose set number?')
-    eq(
-      string.format(
-        [[
-  number
-	Last set from %s line 3]],
-        script_location
-      ),
-      result
-    )
-  end)
 
   it('"Last set" for mapping set by nvim_set_keymap', function()
     local result = exec_capture(':verbose map <leader>key1')
@@ -113,7 +102,7 @@ nohlsearch
         [[
 
 n  \key1       * :echo "test"<CR>
-	Last set from %s line 4]],
+	Last set from %s line 7]],
         script_location
       ),
       result
@@ -127,7 +116,7 @@ n  \key1       * :echo "test"<CR>
         [[
 
 n  \key2       * :echo "test"<CR>
-	Last set from %s line 5]],
+	Last set from %s line 8]],
         script_location
       ),
       result
@@ -142,7 +131,7 @@ n  \key2       * :echo "test"<CR>
 --- Autocommands ---
 test_group  FileType
     c         setl cindent
-	Last set from %s line 7]],
+	Last set from %s line 10]],
         script_location
       ),
       result
@@ -157,7 +146,7 @@ test_group  FileType
 --- Autocommands ---
 test_group  FileType
     cpp       setl cindent
-	Last set from %s line 13]],
+	Last set from %s line 16]],
         script_location
       ),
       result
@@ -170,7 +159,7 @@ test_group  FileType
       string.format(
         [[
 TestHL1        xxx guibg=Blue
-	Last set from %s line 19]],
+	Last set from %s line 22]],
         script_location
       ),
       result
@@ -183,7 +172,7 @@ TestHL1        xxx guibg=Blue
       string.format(
         [[
 TestHL2        xxx guibg=Green
-	Last set from %s line 20]],
+	Last set from %s line 23]],
         script_location
       ),
       result
@@ -200,7 +189,7 @@ TestHL2        xxx guibg=Green
         [[
     Name              Args Address Complete    Definition
     Bdelete           0                        :bd
-	Last set from %s line 22]],
+	Last set from %s line 25]],
         script_location
       ),
       result
@@ -214,7 +203,7 @@ TestHL2        xxx guibg=Green
         [[
     Name              Args Address Complete    Definition
     TestCommand       0                        :echo 'Hello'
-	Last set from %s line 23]],
+	Last set from %s line 26]],
         script_location
       ),
       result
@@ -227,7 +216,7 @@ TestHL2        xxx guibg=Green
       string.format(
         [[
    function Close_Window() abort
-	Last set from %s line 25
+	Last set from %s line 28
 1    wincmd -
    endfunction]],
         script_location
@@ -242,7 +231,7 @@ TestHL2        xxx guibg=Green
       string.format(
         [[
   textwidth=80
-	Last set from %s line 31]],
+	Last set from %s line 34]],
         script_location
       ),
       result
@@ -250,41 +239,89 @@ TestHL2        xxx guibg=Green
   end)
 end
 
-describe('lua :verbose when using :source', function()
+describe('lua :verbose with -V1 when using :source', function()
   last_set_lua_tests('source')
 end)
 
-describe('lua :verbose when using :luafile', function()
+describe('lua :verbose with -V1 when using :luafile', function()
   last_set_lua_tests('luafile')
 end)
 
-describe('lua verbose:', function()
+describe('lua :verbose without -V1', function()
-  local script_file
+  local script_location, script_file
 
+  -- All test cases below use the same Nvim instance.
   setup(function()
     clear()
-    script_file = 'test_luafile.lua'
+    script_file = 'test_verbose_0.lua'
+    local current_dir = fn.getcwd()
+    current_dir = fn.fnamemodify(current_dir, ':~')
+    script_location = table.concat({ current_dir, n.get_pathsep(), script_file })
     write_file(
       script_file,
       [[
-    vim.api.nvim_set_option_value('hlsearch', false, {})
+vim.api.nvim_set_option_value('hlsearch', false, {})
-    ]]
+vim.bo.expandtab = true
+vim.opt.number = true
+vim.api.nvim_exec2('set numberwidth=2', {})
+vim.cmd('set colorcolumn=+1')
+]]
     )
-    exec(':source ' .. script_file)
   end)
 
+  local option_checks = {
+    { 'nvim_set_option_value', 'hlsearch', 'nohlsearch' },
+    { 'vim.bo', 'expandtab', '  expandtab' },
+    { 'vim.opt', 'number', '  number' },
+    { 'nvim_exec2', 'numberwidth', '  numberwidth=2' },
+    { 'vim.cmd', 'colorcolumn', '  colorcolumn=+1' },
+  }
+
   teardown(function()
     os.remove(script_file)
   end)
 
-  it('is disabled when verbose = 0', function()
+  describe('"Last set" shows file name when using :source', function()
-    local result = exec_capture(':verbose set hlsearch?')
+    setup(function()
-    eq(
+      exec(':source ' .. script_file)
-      [[
+    end)
-nohlsearch
+
+    for _, check in ipairs(option_checks) do
+      it(('for option set by %s'):format(check[1]), function()
+        local result = exec_capture((':verbose set %s?'):format(check[2]))
+        eq(
+          string.format(
+            [[
+%s
+	Last set from %s (run Nvim with -V1 for more details)]],
+            check[3],
+            script_location
+          ),
+          result
+        )
+      end)
+    end
+  end)
+
+  describe('"Last set" suggests -V1 when using :luafile', function()
+    setup(function()
+      exec(':luafile ' .. script_file)
+    end)
+
+    for _, check in ipairs(option_checks) do
+      it(('for option set by %s'):format(check[1]), function()
+        local result = exec_capture((':verbose set %s?'):format(check[2]))
+        eq(
+          string.format(
+            [[
+%s
 	Last set from Lua (run Nvim with -V1 for more details)]],
-      result
+            check[3]
-    )
+          ),
+          result
+        )
+      end)
+    end
   end)
 end)
 

test/functional/lua/runtime_spec.lua

@@ -5,6 +5,7 @@ local clear = n.clear
 local eq = t.eq
 local eval = n.eval
 local exec = n.exec
+local api = n.api
 local fn = n.fn
 local mkdir_p = n.mkdir_p
 local rmdir = n.rmdir
@@ -15,9 +16,10 @@ describe('runtime:', function()
   local sep = n.get_pathsep()
   local init = 'dummy_init.lua'
 
+  -- All test cases below use the same Nvim instance.
   setup(function()
     io.open(init, 'w'):close() --  touch init file
-    clear { args = { '-u', init } }
+    clear({ args = { '-u', init } })
     exec('set rtp+=' .. plug_dir)
     exec([[
       set shell=doesnotexist
@@ -39,6 +41,8 @@ describe('runtime:', function()
   after_each(function()
     rmdir(plug_dir)
     exec('bwipe!')
+    exec('set rtp& pp&')
+    exec('set rtp+=' .. plug_dir)
   end)
 
   describe('colors', function()
@@ -404,6 +408,22 @@ describe('runtime:', function()
     end)
   end)
 
+  it('lua file loaded by :runtime has proper script ID #32598', function()
+    local test_file = 'Xtest_runtime_cmd.lua'
+    write_file(
+      table.concat({ plug_dir, test_file }, sep),
+      [[
+      vim.g.script_id = tonumber(vim.fn.expand('<SID>'):match('<SNR>(%d+)_'))
+      vim.o.mouse = 'nv'
+    ]]
+    )
+    exec('runtime ' .. test_file)
+    local expected_sid = fn.getscriptinfo({ name = test_file })[1].sid
+    local sid = api.nvim_get_var('script_id')
+    eq(expected_sid, sid)
+    eq(sid, api.nvim_get_option_info2('mouse', {}).last_set_sid)
+  end)
+
   it('cpp ftplugin loads c ftplugin #29053', function()
     eq('', eval('&commentstring'))
     eq('', eval('&omnifunc'))