fix(lua): allows tables with integer keys to be merged in tbl_deep_extend

- The exclusion of lists was never justified in the commit history and is
  the wrong thing to do for a function that deals with tables.

- Move the error checks out of the recursive path.

Fixes #23654

runtime/doc/news.txt

@@ -210,6 +210,9 @@ These existing features changed their behavior.
   more emoji characters than before, including those encoded with multiple
   emoji codepoints combined with ZWJ (zero width joiner) codepoints.
 
+• |vim.tbl_deep_extend()| no longer ignores any values for which |vim.isarray()|
+  returns `true`.
+
 ==============================================================================
 REMOVED FEATURES                                                 *news-removed*
 

runtime/lua/vim/shared.lua

@@ -354,37 +354,22 @@ function vim.tbl_isempty(t)
   return next(t) == nil
 end
 
---- We only merge empty tables or tables that are not an array (indexed by integers)
+--- Recursive worker for tbl_extend
-local function can_merge(v)
+--- @param behavior 'error'|'keep'|'force'
-  return type(v) == 'table' and (vim.tbl_isempty(v) or not vim.isarray(v))
+--- @param deep_extend boolean
-end
+--- @param ... table<any,any>
-
+local function tbl_extend_rec(behavior, deep_extend, ...)
-local function tbl_extend(behavior, deep_extend, ...)
-  if behavior ~= 'error' and behavior ~= 'keep' and behavior ~= 'force' then
-    error('invalid "behavior": ' .. tostring(behavior))
-  end
-
-  if select('#', ...) < 2 then
-    error(
-      'wrong number of arguments (given '
-        .. tostring(1 + select('#', ...))
-        .. ', expected at least 3)'
-    )
-  end
-
   local ret = {} --- @type table<any,any>
   if vim._empty_dict_mt ~= nil and getmetatable(select(1, ...)) == vim._empty_dict_mt then
     ret = vim.empty_dict()
   end
 
   for i = 1, select('#', ...) do
-    local tbl = select(i, ...)
+    local tbl = select(i, ...) --[[@as table<any,any>]]
-    vim.validate('after the second argument', tbl, 'table')
-    --- @cast tbl table<any,any>
     if tbl then
       for k, v in pairs(tbl) do
-        if deep_extend and can_merge(v) and can_merge(ret[k]) then
+        if deep_extend and type(v) == 'table' and type(ret[k]) == 'table' then
-          ret[k] = tbl_extend(behavior, true, ret[k], v)
+          ret[k] = tbl_extend_rec(behavior, true, ret[k], v)
         elseif behavior ~= 'force' and ret[k] ~= nil then
           if behavior == 'error' then
             error('key found in more than one map: ' .. k)
@@ -395,9 +380,31 @@ local function tbl_extend(behavior, deep_extend, ...)
       end
     end
   end
+
   return ret
 end
 
+--- @param behavior 'error'|'keep'|'force'
+--- @param deep_extend boolean
+--- @param ... table<any,any>
+local function tbl_extend(behavior, deep_extend, ...)
+  if behavior ~= 'error' and behavior ~= 'keep' and behavior ~= 'force' then
+    error('invalid "behavior": ' .. tostring(behavior))
+  end
+
+  local nargs = select('#', ...)
+
+  if nargs < 2 then
+    error(('wrong number of arguments (given %d, expected at least 3)'):format(1 + nargs))
+  end
+
+  for i = 1, nargs do
+    vim.validate('after the second argument', select(i, ...), 'table')
+  end
+
+  return tbl_extend_rec(behavior, deep_extend, ...)
+end
+
 --- Merges two or more tables.
 ---
 ---@see |extend()|

test/functional/lua/vim_spec.lua

@@ -1059,7 +1059,7 @@ describe('lua stdlib', function()
       local a = { a = {[2] = 3} }
       local b = { a = {[3] = 3} }
       local c = vim.tbl_deep_extend("force", a, b)
-      return vim.deep_equal(c, {a = {[3] = 3}})
+      return vim.deep_equal(c, {a = {[2] = 3, [3] = 3}})
     ]]))
 
     eq(
@@ -1071,6 +1071,14 @@ describe('lua stdlib', function()
     ]])
     )
 
+    -- Fix github issue #23654
+    ok(exec_lua([[
+      local a = { sub = { [1] = 'a' } }
+      local b = { sub = { b = 'a' } }
+      local c = vim.tbl_deep_extend('force', a, b)
+      return vim.deep_equal(c, { sub = { [1] = 'a', b = 'a' } })
+    ]]))
+
     matches(
       'invalid "behavior": nil',
       pcall_err(