build: move all generator scripts to src/gen/

- Move all generator Lua scripts to the `src/gen/`
- Add a `.luarc.json` to `src/gen/`
- Add a `preload.lua` to `src/gen/`
  - Add `src` to `package.path` so it aligns with `.luarc.json'
- Fix all `require` statements in `src/gen/` so they are consistent:
    - `require('scripts.foo')` -> `require('gen.foo')`
    - `require('src.nvim.options')` -> `require('nvim.options')`
    - `require('api.dispatch_deprecated')` -> `require('nvim.api.dispatch_deprecated')`

CONTRIBUTING.md

@@ -256,17 +256,17 @@ make lintdoc
 ```
 
 If you need to modify or debug the documentation flow, these are the main files:
-- `./scripts/gen_vimdoc.lua`:
+- `./src/gen/gen_vimdoc.lua`:
   Main doc generator. Parses C and Lua files to render vimdoc files.
-- `./scripts/luacats_parser.lua`:
+- `./src/gen/luacats_parser.lua`:
   Documentation parser for Lua files.
-- `./scripts/cdoc_parser.lua`:
+- `./src/gen/cdoc_parser.lua`:
   Documentation parser for C files.
-- `./scripts/luacats_grammar.lua`:
+- `./src/gen/luacats_grammar.lua`:
   Lpeg grammar for LuaCATS
-- `./scripts/cdoc_grammar.lua`:
+- `./src/gen/cdoc_grammar.lua`:
   Lpeg grammar for C doc comments
-- `./scripts/gen_eval_files.lua`:
+- `./src/gen/gen_eval_files.lua`:
   Generates documentation and Lua type files from metadata files:
   ```
   runtime/lua/vim/*     =>  runtime/doc/lua.txt

cmake/RunTests.cmake

@@ -71,6 +71,7 @@ execute_process(
   COMMAND ${NVIM_PRG} -ll ${WORKING_DIR}/test/lua_runner.lua ${DEPS_INSTALL_DIR} busted -v -o test.busted.outputHandlers.nvim
     --lazy --helper=${TEST_DIR}/${TEST_TYPE}/preload.lua
     --lpath=${BUILD_DIR}/?.lua
+    --lpath=${WORKING_DIR}/src/?.lua
     --lpath=${WORKING_DIR}/runtime/lua/?.lua
     --lpath=?.lua
     ${BUSTED_ARGS}

runtime/CMakeLists.txt

@@ -3,7 +3,7 @@ set(GENERATED_RUNTIME_DIR ${PROJECT_BINARY_DIR}/runtime)
 set(GENERATED_HELP_TAGS ${GENERATED_RUNTIME_DIR}/doc/tags)
 set(GENERATED_PACKAGE_DIR ${GENERATED_RUNTIME_DIR}/pack/dist/opt)
 set(GENERATED_SYN_VIM ${GENERATED_RUNTIME_DIR}/syntax/vim/generated.vim)
-set(SYN_VIM_GENERATOR ${PROJECT_SOURCE_DIR}/src/nvim/generators/gen_vimvim.lua)
+set(SYN_VIM_GENERATOR ${PROJECT_SOURCE_DIR}/src/gen/gen_vimvim.lua)
 
 file(MAKE_DIRECTORY ${GENERATED_RUNTIME_DIR}/syntax/vim)
 

scripts/lintdoc.lua

@@ -14,7 +14,7 @@ print('Running lintdoc ...')
 -- :helptags checks for duplicate tags.
 vim.cmd [[ helptags ALL ]]
 
-require('scripts.gen_help_html').run_validate()
+require('src.gen.gen_help_html').run_validate()
-require('scripts.gen_help_html').test_gen()
+require('src.gen.gen_help_html').test_gen()
 
 print('lintdoc PASSED.')

scripts/release.sh

@@ -59,7 +59,7 @@ _do_release_commit() {
     $__sed -i.bk 's/(NVIM_API_PRERELEASE) true/\1 false/' CMakeLists.txt
     build/bin/nvim --api-info > "test/functional/fixtures/api_level_$__API_LEVEL.mpack"
     git add "test/functional/fixtures/api_level_${__API_LEVEL}.mpack"
-    VIMRUNTIME=./runtime build/bin/nvim -u NONE -l scripts/gen_vimdoc.lua
+    VIMRUNTIME=./runtime build/bin/nvim -u NONE -l src/gen/preload.lua src/gen/gen_vimdoc.lua
     git add -u -- runtime/doc/
   fi
 

src/.luarc.json

@@ -0,0 +1,24 @@
+{
+  "$schema": "https://raw.githubusercontent.com/LuaLS/vscode-lua/master/setting/schema.json",
+  "runtime": {
+    "version": "LuaJIT"
+  },
+  "workspace": {
+    "library": [
+      "../runtime/lua",
+      "${3rd}/luv/library"
+    ],
+    "checkThirdParty": "Disable"
+  },
+  "diagnostics": {
+    "groupFileStatus": {
+      "strict": "Opened",
+      "strong": "Opened"
+    },
+    "groupSeverity": {
+      "strong": "Warning",
+      "strict": "Warning"
+    },
+    "unusedLocalExclude": [ "_*" ]
+  }
+}

src/gen/cdoc_parser.lua

@@ -1,5 +1,5 @@
-local cdoc_grammar = require('scripts.cdoc_grammar')
+local cdoc_grammar = require('gen.cdoc_grammar')
-local c_grammar = require('src.nvim.generators.c_grammar')
+local c_grammar = require('gen.c_grammar')
 
 --- @class nvim.cdoc.parser.param
 --- @field name string

src/gen/gen_api_dispatch.lua

@@ -7,7 +7,7 @@
 
 local mpack = vim.mpack
 
-local hashy = require 'generators.hashy'
+local hashy = require 'gen.hashy'
 
 local pre_args = 7
 assert(#arg >= pre_args)
@@ -31,7 +31,7 @@ local headers = {}
 -- set of function names, used to detect duplicates
 local function_names = {}
 
-local c_grammar = require('generators.c_grammar')
+local c_grammar = require('gen.c_grammar')
 
 local startswith = vim.startswith
 
@@ -150,7 +150,7 @@ end
 
 -- Export functions under older deprecated names.
 -- These will be removed eventually.
-local deprecated_aliases = require('api.dispatch_deprecated')
+local deprecated_aliases = require('nvim.api.dispatch_deprecated')
 for _, f in ipairs(shallowcopy(functions)) do
   local ismethod = false
   if startswith(f.name, 'nvim_') then
@@ -300,7 +300,7 @@ for i, item in ipairs(types) do
 end
 
 local packed = table.concat(pieces)
-local dump_bin_array = require('generators.dump_bin_array')
+local dump_bin_array = require('gen.dump_bin_array')
 dump_bin_array(api_metadata_output, 'packed_api_metadata', packed)
 api_metadata_output:close()
 

src/gen/gen_api_ui_events.lua

@@ -7,10 +7,10 @@ local remote_output = io.open(arg[3], 'wb')
 local metadata_output = io.open(arg[4], 'wb')
 local client_output = io.open(arg[5], 'wb')
 
-local c_grammar = require('generators.c_grammar')
+local c_grammar = require('gen.c_grammar')
 local events = c_grammar.grammar:match(input:read('*all'))
 
-local hashy = require 'generators.hashy'
+local hashy = require 'gen.hashy'
 
 local function write_signature(output, ev, prefix, notype)
   output:write('(' .. prefix)

src/gen/gen_declarations.lua

@@ -1,4 +1,4 @@
-local grammar = require('generators.c_grammar').grammar
+local grammar = require('gen.c_grammar').grammar
 
 --- @param fname string
 --- @return string?

src/gen/gen_eval.lua

@@ -8,7 +8,7 @@ local funcsfname = autodir .. '/funcs.generated.h'
 
 --Will generate funcs.generated.h with definition of functions static const array.
 
-local hashy = require 'generators.hashy'
+local hashy = require 'gen.hashy'
 
 local hashpipe = assert(io.open(funcsfname, 'wb'))
 
@@ -47,7 +47,7 @@ hashpipe:write([[
 
 ]])
 
-local funcs = require('eval').funcs
+local funcs = require('nvim.eval').funcs
 for _, func in pairs(funcs) do
   if func.float_func then
     func.func = 'float_op_wrapper'

src/gen/gen_eval_files.lua

@@ -2,7 +2,7 @@
 
 -- Generator for various vimdoc and Lua type files
 
-local util = require('scripts.util')
+local util = require('gen.util')
 local fmt = string.format
 
 local DEP_API_METADATA = 'build/funcs_metadata.mpack'
@@ -259,7 +259,7 @@ end
 local function get_api_meta()
   local ret = {} --- @type table<string, vim.EvalFn>
 
-  local cdoc_parser = require('scripts.cdoc_parser')
+  local cdoc_parser = require('gen.cdoc_parser')
 
   local f = 'src/nvim/api'
 
@@ -469,7 +469,7 @@ end
 
 --- @return table<string, vim.EvalFn>
 local function get_eval_meta()
-  return require('src/nvim/eval').funcs
+  return require('nvim.eval').funcs
 end
 
 --- Generates LuaLS docstring for a Vimscript "eval" function.
@@ -782,7 +782,7 @@ end
 
 --- @return table<string,vim.option_meta>
 local function get_option_meta()
-  local opts = require('src/nvim/options').options
+  local opts = require('nvim.options').options
   local optinfo = vim.api.nvim_get_all_options_info()
   local ret = {} --- @type table<string,vim.option_meta>
   for _, o in ipairs(opts) do
@@ -806,7 +806,7 @@ end
 
 --- @return table<string,vim.option_meta>
 local function get_vvar_meta()
-  local info = require('src/nvim/vvars').vars
+  local info = require('nvim.vvars').vars
   local ret = {} --- @type table<string,vim.option_meta>
   for name, o in pairs(info) do
     o.desc = dedent(o.desc)

src/gen/gen_events.lua

@@ -1,7 +1,7 @@
 local fileio_enum_file = arg[1]
 local names_file = arg[2]
 
-local auevents = require('auevents')
+local auevents = require('nvim.auevents')
 local events = auevents.events
 
 local enum_tgt = io.open(fileio_enum_file, 'w')

src/gen/gen_ex_cmds.lua

@@ -11,7 +11,7 @@ local enumfile = io.open(enumfname, 'w')
 local defsfile = io.open(defsfname, 'w')
 
 local bit = require 'bit'
-local ex_cmds = require('ex_cmds')
+local ex_cmds = require('nvim.ex_cmds')
 local defs = ex_cmds.cmds
 local flags = ex_cmds.flags
 

src/gen/gen_lsp.lua

@@ -4,9 +4,9 @@ local USAGE = [[
 Generates lua-ls annotations for lsp.
 
 USAGE:
-nvim -l scripts/gen_lsp.lua gen  # by default, this will overwrite runtime/lua/vim/lsp/_meta/protocol.lua
+nvim -l src/gen/gen_lsp.lua gen  # by default, this will overwrite runtime/lua/vim/lsp/_meta/protocol.lua
-nvim -l scripts/gen_lsp.lua gen --version 3.18 --out runtime/lua/vim/lsp/_meta/protocol.lua
+nvim -l src/gen/gen_lsp.lua gen --version 3.18 --out runtime/lua/vim/lsp/_meta/protocol.lua
-nvim -l scripts/gen_lsp.lua gen --version 3.18 --methods --capabilities
+nvim -l src/gen/gen_lsp.lua gen --version 3.18 --methods --capabilities
 ]]
 
 local DEFAULT_LSP_VERSION = '3.18'
@@ -211,13 +211,13 @@ function M.gen(opt)
 
   local output = {
     '--' .. '[[',
-    'THIS FILE IS GENERATED by scripts/gen_lsp.lua',
+    'THIS FILE IS GENERATED by scr/gen/gen_lsp.lua',
     'DO NOT EDIT MANUALLY',
     '',
     'Based on LSP protocol ' .. opt.version,
     '',
     'Regenerate:',
-    ([=[nvim -l scripts/gen_lsp.lua gen --version %s]=]):format(DEFAULT_LSP_VERSION),
+    ([=[nvim -l scr/gen/gen_lsp.lua gen --version %s]=]):format(DEFAULT_LSP_VERSION),
     '--' .. ']]',
     '',
     '---@meta',

src/gen/gen_options.lua

@@ -1,5 +1,5 @@
 --- @module 'nvim.options'
-local options = require('options')
+local options = require('nvim.options')
 local options_meta = options.options
 local cstr = options.cstr
 local valid_scopes = options.valid_scopes
@@ -418,7 +418,7 @@ end
 --- @param option_index table<string,string>
 local function gen_map(output_file, option_index)
   -- Generate option index map.
-  local hashy = require('generators.hashy')
+  local hashy = require('gen.hashy')
 
   local neworder, hashfun = hashy.hashy_hash(
     'find_option',

src/gen/gen_vimdoc.lua

@@ -16,9 +16,9 @@
 ---   parameter is marked as [out].
 --- - Each function documentation is separated by a single line.
 
-local luacats_parser = require('scripts.luacats_parser')
+local luacats_parser = require('gen.luacats_parser')
-local cdoc_parser = require('scripts.cdoc_parser')
+local cdoc_parser = require('gen.cdoc_parser')
-local util = require('scripts.util')
+local util = require('gen.util')
 
 local fmt = string.format
 
@@ -844,7 +844,7 @@ local function get_script_path()
 end
 
 local script_path = get_script_path()
-local base_dir = vim.fs.dirname(vim.fs.dirname(script_path))
+local base_dir = vim.fs.dirname(vim.fs.dirname(vim.fs.dirname(script_path)))
 
 local function delete_lines_below(doc_file, tokenstr)
   local lines = {} --- @type string[]

src/gen/gen_vimvim.lua

@@ -15,9 +15,9 @@ local function w(s)
   end
 end
 
-local options = require('options')
+local options = require('nvim.options')
-local auevents = require('auevents')
+local auevents = require('nvim.auevents')
-local ex_cmds = require('ex_cmds')
+local ex_cmds = require('nvim.ex_cmds')
 
 local function cmd_kw(prev_cmd, cmd)
   if not prev_cmd then

src/gen/luacats_parser.lua

@@ -1,4 +1,4 @@
-local luacats_grammar = require('scripts.luacats_grammar')
+local luacats_grammar = require('gen.luacats_grammar')
 
 --- @class nvim.luacats.parser.param : nvim.luacats.Param
 

src/gen/preload.lua

@@ -0,0 +1,6 @@
+local srcdir = table.remove(arg, 1)
+
+package.path = (srcdir .. '/src/?.lua;') .. (srcdir .. '/runtime/lua/?.lua;') .. package.path
+
+arg[0] = table.remove(arg, 1)
+return loadfile(arg[0])()

src/gen/preload_nlua.lua

@@ -1,8 +1,12 @@
 local srcdir = table.remove(arg, 1)
 local nlualib = table.remove(arg, 1)
 local gendir = table.remove(arg, 1)
-package.path = srcdir .. '/src/nvim/?.lua;' .. srcdir .. '/runtime/lua/?.lua;' .. package.path
+
-package.path = gendir .. '/?.lua;' .. package.path
+package.path = (srcdir .. '/src/?.lua;')
+  .. (srcdir .. '/runtime/lua/?.lua;')
+  .. (gendir .. '/?.lua;')
+  .. package.path
+
 _G.vim = require 'vim.shared'
 _G.vim.inspect = require 'vim.inspect'
 package.cpath = package.cpath .. ';' .. nlualib

src/nvim/CMakeLists.txt

@@ -292,7 +292,7 @@ set(UI_METADATA ${PROJECT_BINARY_DIR}/ui_metadata.mpack)
 set(BINARY_LIB_DIR ${PROJECT_BINARY_DIR}/lib/nvim)
 set(GENERATED_DIR ${PROJECT_BINARY_DIR}/src/nvim/auto)
 set(GENERATED_INCLUDES_DIR ${PROJECT_BINARY_DIR}/include)
-set(GENERATOR_DIR ${CMAKE_CURRENT_LIST_DIR}/generators)
+set(GENERATOR_DIR ${PROJECT_SOURCE_DIR}/src/gen)
 set(GEN_EVAL_TOUCH ${TOUCHES_DIR}/gen_doc_eval)
 set(LUAJIT_RUNTIME_DIR ${DEPS_PREFIX}/share/luajit-2.1/jit)
 set(NVIM_RUNTIME_DIR ${PROJECT_SOURCE_DIR}/runtime)
@@ -306,7 +306,8 @@ set(EX_CMDS_GENERATOR ${GENERATOR_DIR}/gen_ex_cmds.lua)
 set(FUNCS_GENERATOR ${GENERATOR_DIR}/gen_eval.lua)
 set(GENERATOR_C_GRAMMAR ${GENERATOR_DIR}/c_grammar.lua)
 set(GENERATOR_HASHY ${GENERATOR_DIR}/hashy.lua)
-set(GENERATOR_PRELOAD ${GENERATOR_DIR}/preload.lua)
+set(GENERATOR_PRELOAD ${GENERATOR_DIR}/preload_nlua.lua)
+set(NVIM_LUA_PRELOAD ${GENERATOR_DIR}/preload.lua)
 set(HEADER_GENERATOR ${GENERATOR_DIR}/gen_declarations.lua)
 set(OPTIONS_GENERATOR ${GENERATOR_DIR}/gen_options.lua)
 
@@ -514,6 +515,9 @@ add_custom_command(
 set(LUA_GEN ${LUA_GEN_PRG} ${GENERATOR_PRELOAD} ${PROJECT_SOURCE_DIR} $<TARGET_FILE:nlua0> ${PROJECT_BINARY_DIR})
 set(LUA_GEN_DEPS ${GENERATOR_PRELOAD} $<TARGET_FILE:nlua0>)
 
+# Like LUA_GEN but includes also vim.fn, vim.api, vim.uv, etc
+set(NVIM_LUA $<TARGET_FILE:nvim_bin> -u NONE -l ${NVIM_LUA_PRELOAD} ${PROJECT_SOURCE_DIR})
+
 # NVIM_GENERATED_FOR_HEADERS: generated headers to be included in headers
 # NVIM_GENERATED_FOR_SOURCES: generated headers to be included in sources
 # These lists must be mutually exclusive.
@@ -937,12 +941,12 @@ file(GLOB LUA_SOURCES CONFIGURE_DEPENDS
 )
 
 add_target(doc-vim
-  COMMAND $<TARGET_FILE:nvim_bin> -u NONE -l scripts/gen_vimdoc.lua
+  COMMAND ${NVIM_LUA} src/gen/gen_vimdoc.lua
   DEPENDS
     nvim
     ${API_SOURCES}
     ${LUA_SOURCES}
-    ${PROJECT_SOURCE_DIR}/scripts/gen_vimdoc.lua
+    ${PROJECT_SOURCE_DIR}/src/gen/gen_vimdoc.lua
     ${NVIM_RUNTIME_DIR}/doc/api.txt
     ${NVIM_RUNTIME_DIR}/doc/diagnostic.txt
     ${NVIM_RUNTIME_DIR}/doc/lsp.txt
@@ -951,11 +955,11 @@ add_target(doc-vim
   )
 
 add_target(doc-eval
-  COMMAND $<TARGET_FILE:nvim_bin> -u NONE -l ${PROJECT_SOURCE_DIR}/scripts/gen_eval_files.lua
+  COMMAND ${NVIM_LUA} ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua
   DEPENDS
     nvim
     ${FUNCS_METADATA}
-    ${PROJECT_SOURCE_DIR}/scripts/gen_eval_files.lua
+    ${PROJECT_SOURCE_DIR}/src/gen/gen_eval_files.lua
     ${PROJECT_SOURCE_DIR}/src/nvim/eval.lua
     ${PROJECT_SOURCE_DIR}/src/nvim/options.lua
     ${PROJECT_SOURCE_DIR}/src/nvim/vvars.lua
@@ -966,7 +970,7 @@ add_custom_target(doc)
 add_dependencies(doc doc-vim doc-eval)
 
 add_target(lintdoc
-  COMMAND $<TARGET_FILE:nvim_bin> -u NONE -l scripts/lintdoc.lua
+  COMMAND ${NVIM_LUA} scripts/lintdoc.lua
   DEPENDS ${DOCFILES}
   CUSTOM_COMMAND_ARGS USES_TERMINAL)
 add_dependencies(lintdoc nvim)

src/nvim/func_attr.h

@@ -1,6 +1,6 @@
 // Undefined DEFINE_FUNC_ATTRIBUTES and undefined DEFINE_EMPTY_ATTRIBUTES
 // leaves file with untouched FUNC_ATTR_* macros. This variant is used for
-// scripts/gen_declarations.lua.
+// src/gen/gen_declarations.lua.
 //
 // Empty macros are used for *.c files.
 // (undefined DEFINE_FUNC_ATTRIBUTES and defined DEFINE_EMPTY_ATTRIBUTES)

test/.luarc.json

@@ -6,6 +6,7 @@
   "workspace": {
     "library": [
       "../runtime/lua",
+      "../src",
       "../build/usr/share/lua/5.1",
       "../build",
       "${3rd}/busted/library",

test/functional/script/luacats_grammar_spec.lua

@@ -2,7 +2,7 @@ local t = require('test.testutil')
 
 local eq = t.eq
 
-local grammar = require('scripts/luacats_grammar')
+local grammar = require('gen.luacats_grammar')
 
 describe('luacats grammar', function()
   --- @param text string

test/functional/script/luacats_parser_spec.lua

@@ -2,7 +2,7 @@ local t = require('test.testutil')
 
 local eq = t.eq
 
-local parser = require('scripts/luacats_parser')
+local parser = require('gen.luacats_parser')
 
 --- @param name string
 --- @param text string

test/functional/script/text_utils_spec.lua

@@ -11,7 +11,7 @@ local function md_to_vimdoc(text, start_indent, indent, text_width)
     start_indent = start_indent or 0
     indent = indent or 0
     text_width = text_width or 70
-    local util = require('scripts/util')
+    local util = require('src/gen/util')
     return util.md_to_vimdoc(table.concat(text, '\n'), start_indent, indent, text_width)
   ]],
     text,

test/lua_runner.lua

@@ -2,10 +2,8 @@ local platform = vim.uv.os_uname()
 local deps_install_dir = table.remove(_G.arg, 1)
 local subcommand = table.remove(_G.arg, 1)
 local suffix = (platform and platform.sysname:lower():find 'windows') and '.dll' or '.so'
-package.path = deps_install_dir
+package.path = (deps_install_dir .. '/share/lua/5.1/?.lua;')
-  .. '/share/lua/5.1/?.lua;'
+  .. (deps_install_dir .. '/share/lua/5.1/?/init.lua;')
-  .. deps_install_dir
-  .. '/share/lua/5.1/?/init.lua;'
   .. package.path
 package.cpath = deps_install_dir .. '/lib/lua/5.1/?' .. suffix .. ';' .. package.cpath