switch from sqlite to json

This commit is contained in:
atagen 2026-03-23 12:17:45 +11:00
parent 17967da43e
commit f2d0a42fd7
5 changed files with 191 additions and 226 deletions

View file

@ -10,19 +10,17 @@ let usage () =
{|inshellah - nushell completions engine
Usage:
inshellah index PREFIX... [--db PATH]
Index completions into a SQLite database.
inshellah index PREFIX... [--dir PATH]
Index completions into a directory of JSON/nu files.
PREFIX is a directory containing bin/ and share/man/.
Default db: $XDG_CACHE_HOME/inshellah/completions.db
inshellah complete CMD [ARGS...] [--db PATH]
Default dir: $XDG_CACHE_HOME/inshellah
inshellah complete CMD [ARGS...] [--dir PATH] [--system-dir PATH]
Nushell custom completer. Outputs JSON completion candidates.
Falls back to --help resolution if command is not in the database.
inshellah query CMD [--db PATH]
Print stored completion data for CMD as JSON.
inshellah clear [CMD...] [--db PATH]
Clear the database, or remove specific commands.
inshellah dump [--db PATH]
Show stats and commands in the database.
Falls back to --help resolution if command is not indexed.
inshellah query CMD [--dir PATH] [--system-dir PATH]
Print stored completion data for CMD.
inshellah dump [--dir PATH] [--system-dir PATH]
List indexed commands.
inshellah manpage FILE Parse a manpage and emit nushell extern
inshellah manpage-dir DIR Batch-process manpages under DIR
@ -390,9 +388,8 @@ let help_resolve_par ?(timeout=200) cmd rest name =
Each child handles one binary completely (including subcommand resolution)
and marshals results back via pipe. Children use help_resolve_par
which forks per subcommand for parallelism. *)
let cmd_index bindirs mandirs ignorelist db_path =
let db = init db_path in
begin_transaction db;
let cmd_index bindirs mandirs ignorelist dir =
ensure_dir dir;
let done_cmds = ref SSet.empty in
let n_results = ref 0 in
let index_bindir bindir mandir =
@ -418,12 +415,12 @@ let cmd_index bindirs mandirs ignorelist db_path =
try Marshal.from_string data 0 with _ -> `None in
(match result with
| `Native src ->
upsert_raw db ~source:"native" name src;
write_native ~dir name src;
incr n_results
| `Parsed pairs ->
List.iter (fun (cmd_name, r) ->
if not (SSet.mem cmd_name !done_cmds) then begin
upsert db ~source:"help" cmd_name r;
write_result ~dir ~source:"help" cmd_name r;
done_cmds := SSet.add cmd_name !done_cmds;
incr n_results
end
@ -504,7 +501,7 @@ let cmd_index bindirs mandirs ignorelist db_path =
| None -> ()
| Some (cmd, result) ->
if not (SSet.mem cmd !done_cmds) then begin
upsert db ~source:"manpage" cmd result;
write_result ~dir ~source:"manpage" cmd result;
done_cmds := SSet.add cmd !done_cmds;
incr n_results
end
@ -513,23 +510,16 @@ let cmd_index bindirs mandirs ignorelist db_path =
) command_sections
end in
List.iter2 index_bindir bindirs mandirs;
commit db;
Printf.printf "indexed %d commands into %s\n" !n_results db_path;
close db
Printf.printf "indexed %d commands into %s\n" !n_results dir
let cmd_dump db_path =
let db = init db_path in
let (count, sources) = stats db in
Printf.printf "database: %s\n" db_path;
Printf.printf "commands: %d (from %d sources)\n" count sources;
let cmds = all_commands db in
let cmd_dump dirs =
let cmds = all_commands dirs in
Printf.printf "%d commands\n" (List.length cmds);
List.iter (fun cmd ->
match lookup db cmd with
| None -> ()
| Some (_data, source) ->
Printf.printf " %-40s [%s]\n" cmd source
) cmds;
close db
let src = match file_type_of dirs cmd with
| Some s -> s | None -> "?" in
Printf.printf " %-40s [%s]\n" cmd src
) cmds
let find_in_path name =
try
@ -543,10 +533,11 @@ let find_in_path name =
go dirs
with Not_found -> None
let resolve_and_cache db name path =
let resolve_and_cache ~dir name path =
let pairs = help_resolve_par ~timeout:200 path [] name in
if pairs <> [] then begin
List.iter (fun (cmd_name, r) -> upsert db cmd_name r) pairs;
ensure_dir dir;
List.iter (fun (cmd_name, r) -> write_result ~dir cmd_name r) pairs;
Some pairs
end else None
@ -580,18 +571,18 @@ let flag_completions prefix entries =
) entries;
List.rev !candidates
let cmd_complete spans db_path =
let cmd_complete spans user_dir system_dirs =
match spans with
| [] -> print_string "[]\n"
| cmd_name :: rest ->
let db = init db_path in
let dirs = user_dir :: system_dirs in
(* Try longest subcommand match first: "git add" before "git" *)
let rec find_result tokens =
match tokens with
| [] -> None
| _ ->
let try_name = String.concat " " tokens in
match lookup_result db try_name with
match lookup dirs try_name with
| Some r -> Some (try_name, r, List.length tokens)
| None ->
find_result (List.rev (List.tl (List.rev tokens))) in
@ -607,9 +598,8 @@ let cmd_complete spans db_path =
| None ->
(match find_in_path cmd_name with
| Some path ->
(match resolve_and_cache db cmd_name path with
(match resolve_and_cache ~dir:user_dir cmd_name path with
| Some _pairs ->
(* Look up again after caching *)
find_result all_tokens
| None -> None)
| None -> None) in
@ -620,50 +610,25 @@ let cmd_complete spans db_path =
| None -> print_string "[]\n"
| Some (_matched_name, r, _depth) ->
let candidates = ref [] in
(* Flag completions when partial starts with - *)
if String.starts_with ~prefix:"-" partial then
candidates := flag_completions partial r.entries
else begin
(* Subcommand completions *)
List.iter (fun (sc : subcommand) ->
if partial = "" || String.starts_with ~prefix:partial sc.name then
candidates := completion_json sc.name sc.desc :: !candidates
) r.subcommands;
candidates := List.rev !candidates;
(* Also offer flags if no subcommand prefix or few subcommand matches *)
if partial = "" || !candidates = [] then
candidates := !candidates @ flag_completions partial r.entries
end;
Printf.printf "[%s]\n" (String.concat "," !candidates));
close db
Printf.printf "[%s]\n" (String.concat "," !candidates))
let cmd_query cmd db_path =
let db = init db_path in
(match lookup db cmd with
| None ->
Printf.eprintf "not found: %s\n" cmd; close db; exit 1
| Some (data, source) ->
Printf.printf "# source: %s\n%s\n" source data);
close db
let cmd_clear cmds db_path =
let db = init db_path in
(match cmds with
| [] ->
(match Sqlite3.exec db "DELETE FROM completions" with
| Sqlite3.Rc.OK ->
Printf.printf "cleared all commands from %s\n" db_path
| rc ->
Printf.eprintf "error: %s\n" (Sqlite3.Rc.to_string rc); exit 1)
| _ ->
List.iter (fun cmd ->
if has_command db cmd then begin
delete db cmd;
Printf.printf "removed %s\n" cmd
end else
Printf.eprintf "not found: %s\n" cmd
) cmds);
close db
let cmd_query cmd dirs =
match lookup_raw dirs cmd with
| None ->
Printf.eprintf "not found: %s\n" cmd; exit 1
| Some data ->
print_string data; print_newline ()
let load_ignorelist path =
try
@ -679,47 +644,40 @@ let load_ignorelist path =
with _ -> SSet.empty
let parse_index_args args =
let rec go prefixes db ignore = function
| [] -> (List.rev prefixes, db, ignore)
| "--db" :: path :: rest -> go prefixes path ignore rest
| "--ignore" :: path :: rest -> go prefixes db (SSet.union ignore (load_ignorelist path)) rest
| dir :: rest -> go (dir :: prefixes) db ignore rest in
go [] (default_db_path ()) SSet.empty args
let rec go prefixes dir ignore = function
| [] -> (List.rev prefixes, dir, ignore)
| "--dir" :: path :: rest -> go prefixes path ignore rest
| "--ignore" :: path :: rest -> go prefixes dir (SSet.union ignore (load_ignorelist path)) rest
| prefix :: rest -> go (prefix :: prefixes) dir ignore rest in
go [] (default_store_path ()) SSet.empty args
let parse_dir_args args =
let rec go user_dir system_dirs rest_args = function
| [] -> (user_dir, system_dirs, List.rev rest_args)
| "--dir" :: path :: rest -> go path system_dirs rest_args rest
| "--system-dir" :: path :: rest -> go user_dir (path :: system_dirs) rest_args rest
| arg :: rest -> go user_dir system_dirs (arg :: rest_args) rest in
go (default_store_path ()) [] [] args
let () =
match Array.to_list Sys.argv |> List.tl with
| "index" :: rest ->
let (prefixes, db_path, ignorelist) = parse_index_args rest in
let (prefixes, dir, ignorelist) = parse_index_args rest in
if prefixes = [] then (Printf.eprintf "error: index requires at least one prefix dir\n"; exit 1);
let bindirs = List.map (fun p -> Filename.concat p "bin") prefixes in
let mandirs = List.map (fun p -> Filename.concat p "share/man") prefixes in
cmd_index bindirs mandirs ignorelist db_path
cmd_index bindirs mandirs ignorelist dir
| "complete" :: rest ->
let rec parse_complete_args spans db = function
| [] -> (List.rev spans, db)
| "--db" :: path :: rest -> parse_complete_args spans path rest
| arg :: rest -> parse_complete_args (arg :: spans) db rest in
let (spans, db_path) = parse_complete_args [] (default_db_path ()) rest in
cmd_complete spans db_path
let (user_dir, system_dirs, spans) = parse_dir_args rest in
cmd_complete spans user_dir system_dirs
| "query" :: rest ->
let (cmd, db_path) = match rest with
| [cmd] -> (cmd, default_db_path ())
| [cmd; "--db"; path] -> (cmd, path)
| _ -> Printf.eprintf "error: query CMD [--db PATH]\n"; exit 1 in
cmd_query cmd db_path
| "clear" :: rest ->
let rec parse_clear_args cmds db = function
| [] -> (List.rev cmds, db)
| "--db" :: path :: rest -> parse_clear_args cmds path rest
| cmd :: rest -> parse_clear_args (cmd :: cmds) db rest in
let (cmds, db_path) = parse_clear_args [] (default_db_path ()) rest in
cmd_clear cmds db_path
let (user_dir, system_dirs, args) = parse_dir_args rest in
(match args with
| [cmd] -> cmd_query cmd (user_dir :: system_dirs)
| _ -> Printf.eprintf "error: query CMD [--dir PATH] [--system-dir PATH]\n"; exit 1)
| "dump" :: rest ->
let db_path = match rest with
| ["--db"; path] -> path
| [] -> default_db_path ()
| _ -> Printf.eprintf "error: dump [--db PATH]\n"; exit 1 in
cmd_dump db_path
let (user_dir, system_dirs, _) = parse_dir_args rest in
cmd_dump (user_dir :: system_dirs)
| ["manpage"; file] -> cmd_manpage file
| ["manpage-dir"; dir] -> cmd_manpage_dir dir
| _ -> usage ()

View file

@ -20,7 +20,6 @@
angstrom
angstrom-unix
camlzip
ocaml_sqlite3
ppx_inline_test
ocaml-lsp
ocamlformat
@ -44,7 +43,6 @@
angstrom
angstrom-unix
camlzip
ocaml_sqlite3
];
meta.mainProgram = "inshellah";

View file

@ -1,3 +1,3 @@
(library
(name inshellah)
(libraries angstrom angstrom-unix camlzip sqlite3 str unix))
(libraries angstrom angstrom-unix camlzip str unix))

View file

@ -1,35 +1,24 @@
open Parser
let default_db_path () =
let default_store_path () =
let cache = try Sys.getenv "XDG_CACHE_HOME"
with Not_found -> Filename.concat (Sys.getenv "HOME") ".cache" in
Filename.concat cache "inshellah/completions.db"
Filename.concat cache "inshellah"
let ensure_parent path =
let dir = Filename.dirname path in
let ensure_dir dir =
let rec mkdir_p d =
if Sys.file_exists d then ()
else begin mkdir_p (Filename.dirname d); Unix.mkdir d 0o755 end in
mkdir_p dir
let init db_path =
ensure_parent db_path;
let db = Sqlite3.db_open db_path in
let exec sql =
match Sqlite3.exec db sql with
| Sqlite3.Rc.OK -> ()
| rc -> failwith (Printf.sprintf "sqlite: %s: %s" (Sqlite3.Rc.to_string rc) sql) in
exec "PRAGMA journal_mode=WAL";
exec "PRAGMA synchronous=NORMAL";
exec {|CREATE TABLE IF NOT EXISTS completions (
command TEXT PRIMARY KEY,
data TEXT NOT NULL,
source TEXT,
updated_at INTEGER NOT NULL
)|};
db
let filename_of_command cmd =
String.map (function
| ' ' -> '_'
| ('a'..'z' | 'A'..'Z' | '0'..'9' | '-' | '_' | '.') as c -> c
| _ -> '-') cmd
let close db = ignore (Sqlite3.db_close db)
let command_of_filename base =
String.map (function '_' -> ' ' | c -> c) base
(* --- JSON serialization of help_result --- *)
@ -78,8 +67,9 @@ let json_positional_of p =
let json_list f items =
"[" ^ String.concat "," (List.map f items) ^ "]"
let json_of_help_result r =
Printf.sprintf "{\"entries\":%s,\"subcommands\":%s,\"positionals\":%s}"
let json_of_help_result ?(source="help") r =
Printf.sprintf "{\"source\":%s,\"entries\":%s,\"subcommands\":%s,\"positionals\":%s}"
(json_string source)
(json_list json_entry_of r.entries)
(json_list json_subcommand_of r.subcommands)
(json_list json_positional_of r.positionals)
@ -249,96 +239,115 @@ let help_result_of_json j =
subcommands = List.map subcommand_of_json (json_to_list (json_get "subcommands" j));
positionals = List.map positional_of_json (json_to_list (json_get "positionals" j)) }
(* --- Database operations --- *)
(* --- Filesystem operations --- *)
let upsert db ?(source="help") command result =
let json = json_of_help_result result in
let now = int_of_float (Unix.gettimeofday ()) in
let stmt = Sqlite3.prepare db
"INSERT INTO completions (command, data, source, updated_at) VALUES (?, ?, ?, ?)
ON CONFLICT(command) DO UPDATE SET data=excluded.data, source=excluded.source, updated_at=excluded.updated_at" in
ignore (Sqlite3.bind stmt 1 (Sqlite3.Data.TEXT command));
ignore (Sqlite3.bind stmt 2 (Sqlite3.Data.TEXT json));
ignore (Sqlite3.bind stmt 3 (Sqlite3.Data.TEXT source));
ignore (Sqlite3.bind stmt 4 (Sqlite3.Data.INT (Int64.of_int now)));
(match Sqlite3.step stmt with
| Sqlite3.Rc.DONE -> ()
| rc -> failwith (Printf.sprintf "upsert %s: %s" command (Sqlite3.Rc.to_string rc)));
ignore (Sqlite3.finalize stmt)
let write_file path contents =
let oc = open_out path in
output_string oc contents;
close_out oc
let upsert_raw db ?(source="native") command data =
let now = int_of_float (Unix.gettimeofday ()) in
let stmt = Sqlite3.prepare db
"INSERT INTO completions (command, data, source, updated_at) VALUES (?, ?, ?, ?)
ON CONFLICT(command) DO UPDATE SET data=excluded.data, source=excluded.source, updated_at=excluded.updated_at" in
ignore (Sqlite3.bind stmt 1 (Sqlite3.Data.TEXT command));
ignore (Sqlite3.bind stmt 2 (Sqlite3.Data.TEXT data));
ignore (Sqlite3.bind stmt 3 (Sqlite3.Data.TEXT source));
ignore (Sqlite3.bind stmt 4 (Sqlite3.Data.INT (Int64.of_int now)));
(match Sqlite3.step stmt with
| Sqlite3.Rc.DONE -> ()
| rc -> failwith (Printf.sprintf "upsert_raw %s: %s" command (Sqlite3.Rc.to_string rc)));
ignore (Sqlite3.finalize stmt)
let read_file path =
try
let ic = open_in path in
let n = in_channel_length ic in
let s = Bytes.create n in
really_input ic s 0 n;
close_in ic;
Some (Bytes.to_string s)
with _ -> None
let lookup db command =
let stmt = Sqlite3.prepare db
"SELECT data, source FROM completions WHERE command = ?" in
ignore (Sqlite3.bind stmt 1 (Sqlite3.Data.TEXT command));
let result = match Sqlite3.step stmt with
| Sqlite3.Rc.ROW ->
let data = Sqlite3.column_text stmt 0 in
let source = Sqlite3.column_text stmt 1 in
Some (data, source)
| _ -> None in
ignore (Sqlite3.finalize stmt);
result
let write_result ~dir ?(source="help") command result =
let path = Filename.concat dir (filename_of_command command ^ ".json") in
write_file path (json_of_help_result ~source result)
let lookup_result db command =
match lookup db command with
| None -> None
| Some (data, _source) ->
(try Some (help_result_of_json (parse_json data))
with _ -> None)
let write_native ~dir command data =
let path = Filename.concat dir (filename_of_command command ^ ".nu") in
write_file path data
let has_command db command =
let stmt = Sqlite3.prepare db
"SELECT 1 FROM completions WHERE command = ?" in
ignore (Sqlite3.bind stmt 1 (Sqlite3.Data.TEXT command));
let found = Sqlite3.step stmt = Sqlite3.Rc.ROW in
ignore (Sqlite3.finalize stmt);
found
let find_file dirs command =
let base = filename_of_command command in
let rec go = function
| [] -> None
| dir :: rest ->
let json_path = Filename.concat dir (base ^ ".json") in
if Sys.file_exists json_path then Some json_path
else
let nu_path = Filename.concat dir (base ^ ".nu") in
if Sys.file_exists nu_path then Some nu_path
else go rest in
go dirs
let all_commands db =
let stmt = Sqlite3.prepare db "SELECT command FROM completions ORDER BY command" in
let results = ref [] in
while Sqlite3.step stmt = Sqlite3.Rc.ROW do
results := Sqlite3.column_text stmt 0 :: !results
done;
ignore (Sqlite3.finalize stmt);
List.rev !results
let lookup dirs command =
let base = filename_of_command command in
let rec go = function
| [] -> None
| dir :: rest ->
let path = Filename.concat dir (base ^ ".json") in
(match read_file path with
| Some data ->
(try Some (help_result_of_json (parse_json data))
with _ -> None)
| None -> go rest) in
go dirs
let delete db command =
let stmt = Sqlite3.prepare db "DELETE FROM completions WHERE command = ?" in
ignore (Sqlite3.bind stmt 1 (Sqlite3.Data.TEXT command));
ignore (Sqlite3.step stmt);
ignore (Sqlite3.finalize stmt)
let lookup_raw dirs command =
let base = filename_of_command command in
let rec go = function
| [] -> None
| dir :: rest ->
let json_path = Filename.concat dir (base ^ ".json") in
(match read_file json_path with
| Some _ as r -> r
| None ->
let nu_path = Filename.concat dir (base ^ ".nu") in
match read_file nu_path with
| Some _ as r -> r
| None -> go rest) in
go dirs
let begin_transaction db =
match Sqlite3.exec db "BEGIN IMMEDIATE" with
| Sqlite3.Rc.OK -> () | _ -> ()
let has_command dirs command =
find_file dirs command <> None
let commit db =
match Sqlite3.exec db "COMMIT" with
| Sqlite3.Rc.OK -> () | _ -> ()
let all_commands dirs =
let module SSet = Set.Make(String) in
let cmds = ref SSet.empty in
List.iter (fun dir ->
if Sys.file_exists dir && Sys.is_directory dir then
Array.iter (fun f ->
let base =
if Filename.check_suffix f ".json" then
Some (Filename.chop_suffix f ".json")
else if Filename.check_suffix f ".nu" then
Some (Filename.chop_suffix f ".nu")
else None in
match base with
| Some b -> cmds := SSet.add (command_of_filename b) !cmds
| None -> ()
) (Sys.readdir dir)
) dirs;
SSet.elements !cmds
let stats db =
let stmt = Sqlite3.prepare db
"SELECT COUNT(*), COUNT(DISTINCT source) FROM completions" in
let result = match Sqlite3.step stmt with
| Sqlite3.Rc.ROW ->
let count = Sqlite3.column_int stmt 0 in
let sources = Sqlite3.column_int stmt 1 in
(count, sources)
| _ -> (0, 0) in
ignore (Sqlite3.finalize stmt);
result
let delete ~dir command =
let base = filename_of_command command in
let json_path = Filename.concat dir (base ^ ".json") in
let nu_path = Filename.concat dir (base ^ ".nu") in
(try Sys.remove json_path with Sys_error _ -> ());
(try Sys.remove nu_path with Sys_error _ -> ())
let file_type_of dirs command =
let base = filename_of_command command in
let rec go = function
| [] -> None
| dir :: rest ->
let json_path = Filename.concat dir (base ^ ".json") in
if Sys.file_exists json_path then
(match read_file json_path with
| Some data ->
(try Some (json_to_string (json_get "source" (parse_json data)))
with _ -> Some "json")
| None -> Some "json")
else
let nu_path = Filename.concat dir (base ^ ".nu") in
if Sys.file_exists nu_path then Some "native"
else go rest in
go dirs

View file

@ -1,11 +1,12 @@
# NixOS module: automatic nushell completion generation
# NixOS module: automatic nushell completion indexing
#
# Generates completions using three strategies in priority order:
# Indexes completions using three strategies in priority order:
# 1. Native completion generators (e.g. CMD completions nushell)
# 2. Manpage parsing
# 3. --help output parsing
#
# Runs as a single pass during the system profile build.
# Produces a directory of .json/.nu files at build time.
# The `complete` command reads from this directory as a system overlay.
#
# Usage:
# { pkgs, ... }: {
@ -25,20 +26,19 @@ let
in
{
options.programs.inshellah = {
enable = lib.mkEnableOption "nushell completion generation via inshellah";
enable = lib.mkEnableOption "nushell completion indexing via inshellah";
package = lib.mkOption {
type = lib.types.package;
description = "The inshellah package to use for generating completions.";
description = "The inshellah package to use for indexing completions.";
};
generatedCompletionsPath = lib.mkOption {
completionsPath = lib.mkOption {
type = lib.types.str;
default = "/share/nushell/vendor/autoload";
default = "/share/inshellah";
description = ''
Subdirectory within the merged environment where completion files
are placed. The default matches nushell's vendor autoload convention
(discovered via XDG_DATA_DIRS).
are placed. Used as the system-dir for the completer.
'';
};
@ -47,19 +47,19 @@ in
default = [];
example = [ "meat" "problematic-tool" ];
description = ''
List of command names to skip during completion generation.
List of command names to skip during completion indexing.
'';
};
};
config = lib.mkIf cfg.enable {
environment.pathsToLink = [ cfg.generatedCompletionsPath ];
environment.pathsToLink = [ cfg.completionsPath ];
environment.extraSetup =
let
inshellah = "${cfg.package}/bin/inshellah";
destDir = "$out${cfg.generatedCompletionsPath}";
segments = lib.filter (s: s != "") (lib.splitString "/" cfg.generatedCompletionsPath);
destDir = "$out${cfg.completionsPath}";
segments = lib.filter (s: s != "") (lib.splitString "/" cfg.completionsPath);
derefPath = lib.concatMapStringsSep "\n " (seg: ''
_cur="$_cur/${seg}"
if [ -L "$_cur" ]; then
@ -79,10 +79,10 @@ in
${derefPath}
mkdir -p ${destDir}
# Generate all completions in one pass:
# Index completions in one pass:
# native generators > manpages > --help fallback
if [ -d "$out/bin" ] && [ -d "$out/share/man" ]; then
${inshellah} generate "$out/bin" "$out/share/man" -o ${destDir}${ignoreFlag} \
${inshellah} index "$out" --dir ${destDir}${ignoreFlag} \
2>/dev/null || true
fi