fix: update osc52 termfeatures flag on UIEnter/UILeave (#32756)

Problem:

Nvim tries to use OSC 52 even when no TUIs are attached.

Solution:

On each UIEnter/UILeave event, check that there is a TUI client connected to Nvim's stdout.

runtime/plugin/osc52.lua

@@ -1,37 +1,72 @@
-local tty = false
-for _, ui in ipairs(vim.api.nvim_list_uis()) do
-  if ui.chan == 1 and ui.stdout_tty then
-    tty = true
-    break
-  end
-end
+--- @class (private) TermFeatures
+--- @field osc52 boolean?
 
--- Do not query when any of the following is true:
---   * TUI is not attached
---   * OSC 52 support is explicitly disabled via g:termfeatures
---   * Using a badly behaved terminal
-if
-  not tty
-  or (vim.g.termfeatures ~= nil and vim.g.termfeatures.osc52 == false)
-  or vim.env.TERM_PROGRAM == 'Apple_Terminal'
-then
-  return
-end
+local id = vim.api.nvim_create_augroup('nvim.osc52', { clear = true })
+vim.api.nvim_create_autocmd('UIEnter', {
+  group = id,
+  desc = 'Enable OSC 52 feature flag if a supporting TUI is attached',
+  callback = function()
+    -- If OSC 52 is explicitly disabled by the user then don't do anything
+    if vim.g.termfeatures ~= nil and vim.g.termfeatures.osc52 == false then
+      return
+    end
 
-require('vim.termcap').query('Ms', function(cap, found, seq)
-  if not found then
-    return
-  end
+    local tty = false
+    for _, ui in ipairs(vim.api.nvim_list_uis()) do
+      if ui.stdout_tty then
+        tty = true
+        break
+      end
+    end
 
-  assert(cap == 'Ms')
+    -- Do not query when any of the following is true:
+    --   * No TUI is attached
+    --   * Using a badly behaved terminal
+    if not tty or vim.env.TERM_PROGRAM == 'Apple_Terminal' then
+      local termfeatures = vim.g.termfeatures or {} ---@type TermFeatures
+      termfeatures.osc52 = nil
+      vim.g.termfeatures = termfeatures
+      return
+    end
 
-  -- If the terminal reports a sequence other than OSC 52 for the Ms capability
-  -- then ignore it. We only support OSC 52 (for now)
-  if not seq or not seq:match('^\027%]52') then
-    return
-  end
+    require('vim.termcap').query('Ms', function(cap, found, seq)
+      if not found then
+        return
+      end
 
-  local termfeatures = vim.g.termfeatures or {}
-  termfeatures.osc52 = true
-  vim.g.termfeatures = termfeatures
-end)
+      assert(cap == 'Ms')
+
+      -- If the terminal reports a sequence other than OSC 52 for the Ms capability
+      -- then ignore it. We only support OSC 52 (for now)
+      if not seq or not seq:match('^\027%]52') then
+        return
+      end
+
+      local termfeatures = vim.g.termfeatures or {} ---@type TermFeatures
+      termfeatures.osc52 = true
+      vim.g.termfeatures = termfeatures
+    end)
+  end,
+})
+
+vim.api.nvim_create_autocmd('UILeave', {
+  group = id,
+  desc = 'Reset OSC 52 feature flag if no TUIs are attached',
+  callback = function()
+    -- If OSC 52 is explicitly disabled by the user then don't do anything
+    if vim.g.termfeatures ~= nil and vim.g.termfeatures.osc52 == false then
+      return
+    end
+
+    -- If no TUI is connected to Nvim's stdout then reset the OSC 52 term features flag
+    for _, ui in ipairs(vim.api.nvim_list_uis()) do
+      if ui.stdout_tty then
+        return
+      end
+    end
+
+    local termfeatures = vim.g.termfeatures or {} ---@type TermFeatures
+    termfeatures.osc52 = nil
+    vim.g.termfeatures = termfeatures
+  end,
+})

test/functional/terminal/tui_spec.lua

@@ -3329,6 +3329,55 @@ describe('TUI', function()
     retry(nil, 1000, function()
       eq({ true, { osc52 = true } }, { child_session:request('nvim_eval', 'g:termfeatures') })
     end)
+
+    -- Attach another (non-TUI) UI to the child instance
+    local alt = Screen.new(nil, nil, nil, child_session)
+
+    -- Detach the first (primary) client so only the second UI is attached
+    feed_data(':detach\n')
+
+    alt:expect({ any = '%[No Name%]' })
+
+    -- osc52 should be cleared from termfeatures
+    eq({ true, {} }, { child_session:request('nvim_eval', 'g:termfeatures') })
+
+    alt:detach()
+  end)
+
+  it('does not query the terminal for OSC 52 support when disabled', function()
+    clear()
+    exec_lua([[
+      _G.query = false
+      vim.api.nvim_create_autocmd('TermRequest', {
+        callback = function(args)
+          local req = args.data.sequence
+          local sequence = req:match('^\027P%+q([%x;]+)$')
+          if sequence and vim.text.hexdecode(sequence) == 'Ms' then
+            _G.query = true
+          end
+        end,
+      })
+    ]])
+
+    local child_server = new_pipename()
+    screen = tt.setup_child_nvim({
+      '--listen',
+      child_server,
+      -- Use --clean instead of -u NONE to load the osc52 plugin
+      '--clean',
+      '--cmd',
+      'let g:termfeatures = #{osc52: v:false}',
+    }, {
+      env = {
+        VIMRUNTIME = os.getenv('VIMRUNTIME'),
+      },
+    })
+
+    screen:expect({ any = '%[No Name%]' })
+
+    local child_session = n.connect(child_server)
+    eq({ true, { osc52 = false } }, { child_session:request('nvim_eval', 'g:termfeatures') })
+    eq(false, exec_lua([[return _G.query]]))
   end)
 end)