atagen/inshellah
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

init

Browse source
This commit is contained in:
atagen 2026-03-18 15:40:47 +11:00
commit 8060a30534
22 changed files with 3881 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

0
lib/.ocamlformat Normal file
View file

3
lib/dune Normal file
View file

@ -0,0 +1,3 @@
(library
(name inshellah)
(libraries angstrom angstrom-unix camlzip str unix))

911
lib/manpage.ml Normal file
View file

@ -0,0 +1,911 @@
open Parser
(* --- Groff escape/formatting stripper --- *)
let strip_groff_escapes s =
let buf = Buffer.create (String.length s) in
let len = String.length s in
let i = ref 0 in
let last = ref '\000' in
let put c = Buffer.add_char buf c; last := c in
let is_alnum c =
(c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')
in
while !i < len do
if s.[!i] = '\\' && !i + 1 < len then begin
let next = s.[!i + 1] in
match next with
| 'f' ->
(* Font escape: \fB, \fI, \fP, \fR, \f(XX, \f[...] *)
if !i + 2 < len then begin
let fc = s.[!i + 2] in
(* Insert space before italic font to preserve word boundaries
e.g. \fB--max-results\fR\fIcount\fR → "--max-results count" *)
if fc = 'I' && is_alnum !last then put ' ';
if fc = '(' then
i := !i + 5 (* \f(XX *)
else if fc = '[' then begin
i := !i + 3;
while !i < len && s.[!i] <> ']' do incr i done;
if !i < len then incr i
end else
i := !i + 3 (* \fX *)
end else
i := !i + 2
| '-' ->
put '-';
i := !i + 2
| '&' | '/' | ',' ->
(* Zero-width characters *)
i := !i + 2
| '(' ->
(* Two-char named character: \(aq, \(lq, \(rq, etc. *)
if !i + 3 < len then begin
let name = String.sub s (!i + 2) 2 in
(match name with
| "aq" -> put '\''
| "lq" | "Lq" -> put '"'
| "rq" | "Rq" -> put '"'
| "em" | "en" -> put '-'
| _ -> ());
i := !i + 4
end else
i := !i + 2
| '[' ->
(* Named character: \[...] *)
i := !i + 2;
let start = !i in
while !i < len && s.[!i] <> ']' do incr i done;
if !i < len then begin
let name = String.sub s start (!i - start) in
(match name with
| "aq" -> put '\''
| "lq" | "Lq" -> put '"'
| "rq" | "Rq" -> put '"'
| _ -> ());
incr i
end
| 's' ->
(* Size escape: \sN, \s+N, \s-N, \s'N' *)
i := !i + 2;
if !i < len && (s.[!i] = '+' || s.[!i] = '-') then incr i;
if !i < len && s.[!i] >= '0' && s.[!i] <= '9' then incr i;
if !i < len && s.[!i] >= '0' && s.[!i] <= '9' then incr i
| 'm' ->
(* Color escape: \m[...] *)
i := !i + 2;
if !i < len && s.[!i] = '[' then begin
incr i;
while !i < len && s.[!i] <> ']' do incr i done;
if !i < len then incr i
end
| 'X' ->
(* Device control: \X'...' *)
i := !i + 2;
if !i < len && s.[!i] = '\'' then begin
incr i;
while !i < len && s.[!i] <> '\'' do incr i done;
if !i < len then incr i
end
| '*' ->
(* String variable: \*X or \*(XX or \*[...] *)
i := !i + 2;
if !i < len then begin
if s.[!i] = '(' then
i := !i + 2
else if s.[!i] = '[' then begin
incr i;
while !i < len && s.[!i] <> ']' do incr i done;
if !i < len then incr i
end else
incr i
end
| 'n' ->
(* Number register: \nX or \n(XX or \n[...] *)
i := !i + 2;
if !i < len then begin
if s.[!i] = '(' then
i := !i + 2
else if s.[!i] = '[' then begin
incr i;
while !i < len && s.[!i] <> ']' do incr i done;
if !i < len then incr i
end else
incr i
end
| 'e' ->
put '\\';
i := !i + 2
| '\\' ->
put '\\';
i := !i + 2
| ' ' ->
put ' ';
i := !i + 2
| _ ->
(* Unknown escape, skip *)
i := !i + 2
end else begin
put s.[!i];
incr i
end
done;
Buffer.contents buf
(* Strip inline macro formatting: .BI, .BR, .IR, etc.
These macros alternate between fonts for their arguments.
We just concatenate the arguments. *)
let strip_inline_macro_args s =
(* Arguments are separated by spaces, quoted strings are kept together *)
let buf = Buffer.create (String.length s) in
let len = String.length s in
let i = ref 0 in
while !i < len do
if s.[!i] = '"' then begin
incr i;
while !i < len && s.[!i] <> '"' do
Buffer.add_char buf s.[!i];
incr i
done;
if !i < len then incr i
end else if s.[!i] = ' ' || s.[!i] = '\t' then begin
incr i
end else begin
Buffer.add_char buf s.[!i];
incr i
end
done;
Buffer.contents buf
let strip_groff line =
let s = strip_groff_escapes line in
String.trim s
(* --- Line classification --- *)
type groff_line =
| Macro of string * string (* e.g. ("SH", "OPTIONS") or ("TP", "") *)
| Text of string (* plain text after stroff stripping *)
| Blank
| Comment
let classify_line line =
let len = String.length line in
if len = 0 then Blank
else if len >= 2 && line.[0] = '.' && line.[1] = '\\' && (len < 3 || line.[2] = '"') then
Comment
else if len >= 3 && line.[0] = '\\' && line.[1] = '"' then
Comment
else if line.[0] = '.' || line.[0] = '\'' then begin
(* Macro line *)
let rest = String.sub line 1 (len - 1) in
let rest = String.trim rest in
(* Split into macro name and arguments *)
let space_pos =
try Some (String.index rest ' ')
with Not_found ->
try Some (String.index rest '\t')
with Not_found -> None
in
match space_pos with
| Some pos ->
let name = String.sub rest 0 pos in
let args = String.trim (String.sub rest (pos + 1) (String.length rest - pos - 1)) in
(* Strip quotes from args *)
let args =
let alen = String.length args in
if alen >= 2 && args.[0] = '"' && args.[alen - 1] = '"' then
String.sub args 1 (alen - 2)
else args
in
Macro (name, args)
| None ->
Macro (rest, "")
end else begin
let stripped = strip_groff line in
if String.length stripped = 0 then Blank
else Text stripped
end
(* Check for dot-backslash-quote style comments more carefully *)
let is_comment_line line =
let len = String.length line in
(len >= 3 && line.[0] = '.' && line.[1] = '\\' && line.[2] = '"')
|| (len >= 2 && line.[0] = '\\' && line.[1] = '"')
let classify_line line =
if is_comment_line line then Comment
else classify_line line
(* --- Section extraction --- *)
let extract_options_section lines =
let classified = List.map classify_line lines in
let rec collect_until_next_sh lines acc =
match lines with
| [] -> List.rev acc
| Macro ("SH", _) :: _ -> List.rev acc
| line :: rest -> collect_until_next_sh rest (line :: acc)
in
let is_options_section name =
let s = String.uppercase_ascii (String.trim name) in
s = "OPTIONS"
|| (String.length s > 0 &&
try let _ = Str.search_forward (Str.regexp_string "OPTION") s 0 in true
with Not_found -> false)
in
(* First pass: look for OPTIONS section *)
let rec find_options = function
| [] -> None
| Macro ("SH", args) :: rest when is_options_section args ->
Some (collect_until_next_sh rest [])
| _ :: rest -> find_options rest
in
(* Fallback: DESCRIPTION section *)
let rec find_description = function
| [] -> []
| Macro ("SH", args) :: rest
when String.uppercase_ascii (String.trim args) = "DESCRIPTION" ->
collect_until_next_sh rest []
| _ :: rest -> find_description rest
in
match find_options classified with
| Some section -> section
| None -> find_description classified
(* --- Strategy-based entry extraction --- *)
(* Collect text lines until next macro or blank *)
let rec collect_text_lines lines acc =
match lines with
| Text s :: rest -> collect_text_lines rest (s :: acc)
| _ -> (String.concat " " (List.rev acc), lines)
(* Parse a tag line to extract entry using the Angstrom switch_parser *)
let parse_tag_to_entry tag desc =
let tag = strip_groff_escapes tag in
let tag = String.trim tag in
match Angstrom.parse_string ~consume:Angstrom.Consume.Prefix
(Angstrom.lift2 (fun sw p -> (sw, p)) switch_parser param_parser) tag with
| Ok (switch, param) -> Some { switch; param; desc }
| Error _ -> None
(* Extract tag text from a macro line (.B, .I preserve spaces; .BI/.BR/.IR alternate) *)
let tag_of_macro name args =
match name with
| "B" | "I" -> strip_groff_escapes args |> String.trim
| _ -> strip_inline_macro_args args |> strip_groff_escapes |> String.trim
(* Strategy A: .TP style (most common — GNU coreutils, help2man) *)
let strategy_tp lines =
let rec walk lines acc =
match lines with
| [] -> List.rev acc
| Macro ("TP", _) :: rest ->
(* Next line is the tag — could be Text or a formatting macro *)
begin match rest with
| Text tag :: rest2 ->
let (desc, rest3) = collect_text_lines rest2 [] in
let entry = parse_tag_to_entry tag desc in
walk rest3 (match entry with Some e -> e :: acc | None -> acc)
| Macro (("B" | "I" | "BI" | "BR" | "IR") as m, args) :: rest2 ->
let tag = tag_of_macro m args in
let (desc, rest3) = collect_text_lines rest2 [] in
let entry = parse_tag_to_entry tag desc in
walk rest3 (match entry with Some e -> e :: acc | None -> acc)
| _ -> walk rest acc
end
| _ :: rest -> walk rest acc
in
walk lines []
(* Strategy B: .IP style (curl, hand-written) *)
let strategy_ip lines =
let rec walk lines acc =
match lines with
| [] -> List.rev acc
| Macro ("IP", tag) :: rest ->
let tag = strip_groff_escapes tag in
let (desc, rest2) = collect_text_lines rest [] in
let entry = parse_tag_to_entry tag desc in
walk rest2 (match entry with Some e -> e :: acc | None -> acc)
| _ :: rest -> walk rest acc
in
walk lines []
(* Strategy C: .PP + .RS/.RE style (git, DocBook) *)
let strategy_pp_rs lines =
let rec walk lines acc =
match lines with
| [] -> List.rev acc
| Macro ("PP", _) :: rest ->
begin match rest with
| Text tag :: rest2 ->
(* Look for .RS ... text ... .RE *)
let rec collect_rs lines desc_acc =
match lines with
| Macro ("RS", _) :: rest3 ->
collect_in_rs rest3 desc_acc
| Text s :: rest3 ->
(* Sometimes description follows directly *)
collect_rs rest3 (s :: desc_acc)
| _ -> (String.concat " " (List.rev desc_acc), lines)
and collect_in_rs lines desc_acc =
match lines with
| Macro ("RE", _) :: rest3 ->
(String.concat " " (List.rev desc_acc), rest3)
| Text s :: rest3 ->
collect_in_rs rest3 (s :: desc_acc)
| Macro ("PP", _) :: _ | Macro ("SH", _) :: _ ->
(String.concat " " (List.rev desc_acc), lines)
| _ :: rest3 -> collect_in_rs rest3 desc_acc
| [] -> (String.concat " " (List.rev desc_acc), [])
in
let (desc, rest3) = collect_rs rest2 [] in
let entry = parse_tag_to_entry tag desc in
walk rest3 (match entry with Some e -> e :: acc | None -> acc)
| _ -> walk rest acc
end
| _ :: rest -> walk rest acc
in
walk lines []
(* Strategy D: Deroff fallback — strip all groff, use help text parser *)
let strategy_deroff_lines lines =
let buf = Buffer.create 256 in
List.iter (fun line ->
match line with
| Text s ->
Buffer.add_string buf s;
Buffer.add_char buf '\n'
| Macro (("BI" | "BR" | "IR" | "B" | "I"), args) ->
let text = strip_inline_macro_args args in
let text = strip_groff_escapes text in
Buffer.add_string buf text;
Buffer.add_char buf '\n'
| Blank -> Buffer.add_char buf '\n'
| _ -> ()
) lines;
let text = Buffer.contents buf in
match parse_help text with
| Ok result -> result.entries
| Error _ -> []
(* Strategy E: Nix3-style bullet .IP with .UR/.UE hyperlinks *)
let strategy_nix lines =
let is_bullet_ip args =
String.length (String.trim args) > 0
in
let rec walk lines acc =
match lines with
| [] -> List.rev acc
| Macro ("IP", args) :: rest when is_bullet_ip args ->
(* Collect tag: skip UR/UE macros, collect Text lines *)
let rec collect_tag lines parts =
match lines with
| Macro ("UR", _) :: rest2 -> collect_tag rest2 parts
| Macro ("UE", _) :: rest2 -> collect_tag rest2 parts
| Text s :: rest2 -> collect_tag rest2 (s :: parts)
| _ -> (String.concat " " (List.rev parts), lines)
in
let (tag, rest2) = collect_tag rest [] in
(* Collect description after the description .IP marker *)
let rec collect_desc lines parts =
match lines with
| Macro ("IP", dargs) :: rest3 when not (is_bullet_ip dargs) ->
collect_desc_text rest3 parts
| _ -> (String.concat " " (List.rev parts), lines)
and collect_desc_text lines parts =
match lines with
| Text s :: rest3 -> collect_desc_text rest3 (s :: parts)
| Macro ("IP", args) :: _ when is_bullet_ip args ->
(String.concat " " (List.rev parts), lines)
| Macro (("SS" | "SH"), _) :: _ ->
(String.concat " " (List.rev parts), lines)
| Macro ("RS", _) :: rest3 ->
skip_rs rest3 parts 1
| Macro ("IP", _) :: rest3 ->
(* Non-bullet IP = continuation paragraph *)
collect_desc_text rest3 parts
| Macro _ :: rest3 -> collect_desc_text rest3 parts
| Blank :: rest3 -> collect_desc_text rest3 parts
| Comment :: rest3 -> collect_desc_text rest3 parts
| [] -> (String.concat " " (List.rev parts), [])
and skip_rs lines parts depth =
match lines with
| Macro ("RE", _) :: rest3 ->
if depth <= 1 then collect_desc_text rest3 parts
else skip_rs rest3 parts (depth - 1)
| Macro ("RS", _) :: rest3 -> skip_rs rest3 parts (depth + 1)
| _ :: rest3 -> skip_rs rest3 parts depth
| [] -> (String.concat " " (List.rev parts), [])
in
let (desc, rest3) = collect_desc rest2 [] in
let entry = parse_tag_to_entry tag desc in
walk rest3 (match entry with Some e -> e :: acc | None -> acc)
| _ :: rest -> walk rest acc
in
walk lines []
(* Count macros of a given type *)
let count_macro name lines =
List.fold_left (fun n line ->
match line with Macro (m, _) when m = name -> n + 1 | _ -> n
) 0 lines
(* Auto-detect and try strategies, return the one with most entries *)
let extract_entries lines =
let tp = count_macro "TP" lines
and ip = count_macro "IP" lines
and pp = count_macro "PP" lines
and rs = count_macro "RS" lines
and ur = count_macro "UR" lines in
let specialized = List.filter_map Fun.id [
(if tp > 0 then Some ("TP", strategy_tp lines) else None);
(if ip > 0 then Some ("IP", strategy_ip lines) else None);
(if pp > 0 && rs > 0 then Some ("PP+RS", strategy_pp_rs lines) else None);
(if ur > 0 && ip > 0 then Some ("nix", strategy_nix lines) else None);
] in
let candidates = match List.filter (fun (_, e) -> e <> []) specialized with
| [] -> [("deroff", strategy_deroff_lines lines)]
| filtered -> filtered
in
List.fold_left (fun (_, best) (name, entries) ->
if List.length entries >= List.length best then (name, entries)
else (name, best)
) ("none", []) candidates |> snd
(* --- NAME section description extraction --- *)
let extract_name_description contents =
let lines = String.split_on_char '\n' contents in
let classified = List.map classify_line lines in
let rec find = function
| [] -> None
| Macro ("SH", args) :: rest
when String.uppercase_ascii (String.trim args) = "NAME" ->
collect rest []
| _ :: rest -> find rest
and collect lines acc =
match lines with
| Macro ("SH", _) :: _ | [] -> finish acc
| Text s :: rest -> collect rest (s :: acc)
| Macro (("B" | "BI" | "BR" | "I" | "IR"), args) :: rest ->
let s = strip_inline_macro_args args |> strip_groff_escapes |> String.trim in
collect rest (if String.length s > 0 then s :: acc else acc)
| Macro ("Nm", args) :: rest ->
let s = strip_groff_escapes args |> String.trim in
collect rest (if String.length s > 0 then s :: acc else acc)
| Macro ("Nd", args) :: rest ->
let s = strip_groff_escapes args |> String.trim in
collect rest (if String.length s > 0 then ("\\- " ^ s) :: acc else acc)
| _ :: rest -> collect rest acc
and finish acc =
let full = String.concat " " (List.rev acc) |> String.trim in
(* NAME lines look like: "git-add \- Add file contents to the index" *)
let sep = Str.regexp {| *\\- *\| +- +|} in
match Str.bounded_split sep full 2 with
| [_; desc] -> Some (String.trim desc)
| _ -> None
in
find classified
(* --- SYNOPSIS command name extraction --- *)
let extract_synopsis_command_lines lines =
let classified = List.map classify_line lines in
let is_synopsis name =
let s = String.uppercase_ascii (String.trim name) in
s = "SYNOPSIS"
in
let extract_cmd line =
let words = String.split_on_char ' ' (String.trim line) in
let words = List.filter (fun w -> String.length w > 0) words in
let is_cmd_char = function
| 'a'..'z' | 'A'..'Z' | '0'..'9' | '-' | '_' | '.' -> true
| _ -> false
in
let rec take = function
| [] -> []
| w :: rest ->
if String.length w > 0
&& (w.[0] = '[' || w.[0] = '-' || w.[0] = '<'
|| w.[0] = '(' || w.[0] = '{')
then []
else if String.for_all is_cmd_char w then
w :: take rest
else []
in
match take words with
| [] -> None
| cmd -> Some (String.concat " " cmd)
in
let rec find = function
| [] -> None
| Macro ("SH", args) :: rest when is_synopsis args -> collect rest
| _ :: rest -> find rest
and collect = function
| [] -> None
| Macro ("SH", _) :: _ -> None
| Text s :: _ ->
let s = String.trim s in
if String.length s > 0 then extract_cmd s else None
| Macro (("B" | "BI" | "BR"), args) :: _ ->
let s = strip_inline_macro_args args |> strip_groff_escapes |> String.trim in
if String.length s > 0 then extract_cmd s else None
| _ :: rest -> collect rest
in
find classified
let extract_synopsis_command contents =
let lines = String.split_on_char '\n' contents in
extract_synopsis_command_lines lines
(* --- SYNOPSIS positional extraction --- *)
let extract_synopsis_positionals_lines lines =
let classified = List.map classify_line lines in
let is_synopsis name =
String.uppercase_ascii (String.trim name) = "SYNOPSIS"
in
let rec find = function
| [] -> []
| Macro ("SH", args) :: rest when is_synopsis args -> collect rest []
| _ :: rest -> find rest
and collect lines acc =
match lines with
| [] -> finish acc
| Macro ("SH", _) :: _ -> finish acc
| Macro ("SS", _) :: _ -> finish acc
| Macro ("br", _) :: _ -> finish acc
| Text s :: rest ->
let s = strip_groff_escapes s |> String.trim in
collect rest (if String.length s > 0 then s :: acc else acc)
| Macro (("B" | "BI" | "BR" | "I" | "IR" | "IB" | "RB" | "RI"), args) :: rest ->
let s = strip_inline_macro_args args |> strip_groff_escapes |> String.trim in
collect rest (if String.length s > 0 then s :: acc else acc)
| _ :: rest -> collect rest acc
and finish acc =
let parts = List.rev acc in
let full = String.concat " " parts |> String.trim in
if String.length full = 0 then []
else
let cmd_end = skip_command_prefix full in
let args = String.sub full cmd_end (String.length full - cmd_end) in
parse_usage_args args
in
find classified
(* --- mdoc (BSD) format support --- *)
let is_mdoc lines =
List.exists (fun l ->
match classify_line l with Macro ("Sh", _) -> true | _ -> false
) lines
let mdoc_text_of line =
match line with
| Text s -> Some (strip_groff_escapes s)
| Macro (m, args) ->
(match m with
| "Pp" | "Bl" | "El" | "Sh" | "Ss" | "Os" | "Dd" | "Dt"
| "Oo" | "Oc" | "Op" -> None
| _ ->
let s = strip_groff_escapes args |> String.trim in
if s = "" then None else Some s)
| _ -> None
let parse_mdoc_it args =
let words = String.split_on_char ' ' args
|> List.filter (fun w -> w <> "" && w <> "Ns") in
let param = match words with
| _ :: _ :: "Ar" :: p :: _ -> Some (Mandatory p)
| _ -> None
in
match words with
| "Fl" :: c :: _ when String.length c = 1 && is_alphanumeric c.[0] ->
Some { switch = Short c.[0]; param; desc = "" }
| "Fl" :: name :: _ when String.length name > 1 && name.[0] = '-' ->
Some { switch = Long (String.sub name 1 (String.length name - 1)); param; desc = "" }
| _ -> None
let positional_of_mdoc_line optional args =
let words = String.split_on_char ' ' args
|> List.filter (fun w -> w <> "") in
match words with
| name :: _ when String.length name >= 2 ->
Some { pos_name = String.lowercase_ascii name;
optional; variadic = List.mem "..." words }
| _ -> None
let parse_mdoc_lines lines =
let classified = List.map classify_line lines in
let rec skip_to_el = function
| [] -> []
| Macro ("El", _) :: rest -> rest
| _ :: rest -> skip_to_el rest
in
let rec collect_desc acc = function
| [] -> (acc, [])
| (Macro ("It", _) | Macro ("El", _)
| Macro ("Sh", _) | Macro ("Ss", _)) :: _ as rest -> (acc, rest)
| line :: rest ->
collect_desc (match mdoc_text_of line with Some s -> s :: acc | None -> acc) rest
in
let desc_of rest =
let parts, rest = collect_desc [] rest in
(String.concat " " (List.rev parts) |> String.trim, rest)
in
let parse_it args rest entries =
let desc, rest = desc_of rest in
let entries = match parse_mdoc_it args with
| Some e -> { e with desc } :: entries
| None -> entries
in
(entries, rest)
in
let rec parse_option_list entries = function
| [] -> (entries, [])
| Macro ("El", _) :: rest -> (entries, rest)
| Macro ("It", args) :: rest ->
let entries, rest = parse_it args rest entries in
parse_option_list entries rest
| _ :: rest -> parse_option_list entries rest
in
let rec scan entries positionals = function
| [] -> (entries, positionals)
| Macro ("Bl", _) :: Macro ("It", it_args) :: rest ->
let words = String.split_on_char ' ' it_args
|> List.filter (fun w -> w <> "") in
if (match words with "Fl" :: _ -> true | _ -> false) then
let entries, rest = parse_it it_args rest entries in
let entries, rest = parse_option_list entries rest in
scan entries positionals rest
else
scan entries positionals (skip_to_el rest)
| Macro ("Bl", _) :: rest -> scan entries positionals (skip_to_el rest)
| Macro ("Sh", args) :: rest
when String.uppercase_ascii (String.trim args) = "SYNOPSIS" ->
let positionals, rest = parse_synopsis positionals rest in
scan entries positionals rest
| _ :: rest -> scan entries positionals rest
and parse_synopsis positionals = function
| [] -> (positionals, [])
| Macro ("Sh", _) :: _ as rest -> (positionals, rest)
| Macro ("Ar", args) :: rest ->
let positionals = match positional_of_mdoc_line false args with
| Some p -> p :: positionals | None -> positionals in
parse_synopsis positionals rest
| Macro ("Op", args) :: rest ->
let words = String.split_on_char ' ' args
|> List.filter (fun w -> w <> "") in
let positionals = match words with
| "Ar" :: _ ->
(match positional_of_mdoc_line true args with
| Some p -> p :: positionals | None -> positionals)
| _ -> positionals in
parse_synopsis positionals rest
| _ :: rest -> parse_synopsis positionals rest
in
let entries, positionals = scan [] [] classified in
let positionals =
List.rev positionals
|> List.fold_left (fun (seen, acc) p ->
if List.mem p.pos_name seen then (seen, acc)
else (p.pos_name :: seen, p :: acc)
) ([], [])
|> snd |> List.rev
in
{ entries = List.rev entries; subcommands = []; positionals; description = "" }
(* --- COMMANDS section subcommand extraction --- *)
(* Extract subcommands from COMMANDS/COMMAND sections.
These use .PP + bold name + .RS/.RE blocks, e.g.:
.PP
\fBstart\fR \fIUNIT\fR...
.RS 4
Start (activate) one or more units.
.RE *)
let extract_commands_section lines =
let classified = List.map classify_line lines in
let rec collect_until_next_sh lines acc =
match lines with
| [] -> List.rev acc
| Macro ("SH", _) :: _ -> List.rev acc
| line :: rest -> collect_until_next_sh rest (line :: acc)
in
let is_commands_section name =
let s = String.uppercase_ascii (String.trim name) in
s = "COMMANDS" || s = "COMMAND"
in
let rec find_commands acc = function
| [] -> List.rev acc
| Macro ("SH", args) :: rest when is_commands_section args ->
find_commands (collect_until_next_sh rest [] :: acc) rest
| _ :: rest -> find_commands acc rest
in
let sections = find_commands [] classified in
List.concat sections
(* Extract subcommand name from a bold groff text like
"\fBlist\-units\fR [\fIPATTERN\fR...]" → "list-units" *)
let extract_bold_command_name text =
let s = String.trim text in
(* Look for \fB...\fR at the start *)
if String.length s >= 4
&& s.[0] = '\\' && s.[1] = 'f' && s.[2] = 'B' then
let start = 3 in
let end_marker = "\\fR" in
match String.split_on_char '\\' (String.sub s start (String.length s - start)) with
| name_part :: _ ->
let name = strip_groff_escapes ("\\fB" ^ name_part ^ end_marker) |> String.trim in
(* Must look like a subcommand: lowercase, hyphens, no leading dash *)
if String.length name >= 2
&& name.[0] <> '-'
&& String.for_all (fun c ->
(c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c = '-' || c = '_'
) name
then Some name
else None
| [] -> None
else
(* Try already-stripped text *)
let stripped = strip_groff_escapes s in
let first_word = match String.split_on_char ' ' stripped with
| w :: _ -> w | [] -> "" in
if String.length first_word >= 2
&& first_word.[0] <> '-'
&& String.for_all (fun c ->
(c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c = '-' || c = '_'
) first_word
then Some first_word
else None
let extract_subcommands_from_commands lines =
let rec walk lines acc =
match lines with
| [] -> List.rev acc
| Macro ("PP", _) :: rest ->
begin match rest with
| Text tag :: rest2 ->
(* Check if this is a subcommand (bold name, not a flag) *)
begin match extract_bold_command_name tag with
| Some name ->
(* Collect description from .RS/.RE block *)
let rec collect_desc lines desc_acc =
match lines with
| Macro ("RS", _) :: rest3 ->
collect_in_rs rest3 desc_acc
| Text s :: rest3 ->
collect_desc rest3 (s :: desc_acc)
| _ -> (String.concat " " (List.rev desc_acc), lines)
and collect_in_rs lines desc_acc =
match lines with
| Macro ("RE", _) :: rest3 ->
(String.concat " " (List.rev desc_acc), rest3)
| Text s :: rest3 ->
collect_in_rs rest3 (s :: desc_acc)
| Macro ("PP", _) :: _ | Macro ("SH", _) :: _ | Macro ("SS", _) :: _ ->
(String.concat " " (List.rev desc_acc), lines)
| _ :: rest3 -> collect_in_rs rest3 desc_acc
| [] -> (String.concat " " (List.rev desc_acc), [])
in
let (desc, rest3) = collect_desc rest2 [] in
let desc = String.trim desc in
(* Take first sentence as description *)
let short_desc = match String.split_on_char '.' desc with
| first :: _ when String.length first > 0 -> String.trim first
| _ -> desc in
let sc : subcommand = { name; desc = short_desc } in
walk rest3 (sc :: acc)
| None -> walk rest2 acc
end
| _ -> walk rest acc
end
| _ :: rest -> walk rest acc
in
walk lines []
(* --- Top-level API --- *)
let parse_manpage_lines lines =
if is_mdoc lines then
parse_mdoc_lines lines
else begin
let options_section = extract_options_section lines in
let entries = extract_entries options_section in
let positionals = extract_synopsis_positionals_lines lines in
let commands_section = extract_commands_section lines in
let subcommands = extract_subcommands_from_commands commands_section in
{ entries; subcommands; positionals; description = "" }
end
let parse_manpage_string contents =
let lines = String.split_on_char '\n' contents in
let result = parse_manpage_lines lines in
let description = match extract_name_description contents with
| Some d -> d | None -> "" in
{ result with description }
(* --- Clap-style SUBCOMMAND section extraction --- *)
(* Manpages generated by clap (Rust) put each subcommand under its own
.SH SUBCOMMAND header with a Usage: line giving the name. *)
let extract_subcommand_sections contents =
let lines = String.split_on_char '\n' contents in
let classified = List.map classify_line lines in
(* Split into sections at .SH boundaries *)
let rec collect_sections acc current_name current_lines = function
| [] ->
let acc = match current_name with
| Some n -> (n, List.rev current_lines) :: acc
| None -> acc in
List.rev acc
| Macro ("SH", args) :: rest ->
let acc = match current_name with
| Some n -> (n, List.rev current_lines) :: acc
| None -> acc in
let name = String.uppercase_ascii (String.trim args) in
if name = "SUBCOMMAND" || name = "SUBCOMMANDS" then
collect_sections acc (Some name) [] rest
else
collect_sections acc None [] rest
| line :: rest ->
collect_sections acc current_name (line :: current_lines) rest
in
let sections = collect_sections [] None [] classified in
(* For each SUBCOMMAND section, extract name from Usage: line and parse entries *)
let usage_re = Str.regexp {|Usage: \([a-zA-Z0-9_-]+\)|} in
let matches_usage s =
try ignore (Str.search_forward usage_re s 0); Some (Str.matched_group 1 s)
with Not_found -> None in
List.filter_map (fun (_header, section_lines) ->
let name, desc_lines =
List.fold_left (fun (name, desc_lines) line ->
match name with
| Some _ -> (name, desc_lines)
| None ->
match line with
| Text s ->
(match matches_usage s with
| Some _ as found -> (found, desc_lines)
| None -> (None, s :: desc_lines))
| Macro (("TP" | "B" | "BI" | "BR"), args) ->
let s = strip_inline_macro_args args |> strip_groff_escapes |> String.trim in
(matches_usage s, desc_lines)
| _ -> (None, desc_lines)
) (None, []) section_lines in
match name with
| None -> None
| Some subcmd_name ->
let entries = extract_entries section_lines in
let desc = String.concat " " (List.rev desc_lines)
|> strip_groff_escapes |> String.trim in
let desc = Str.global_replace (Str.regexp "`\\([^`]*\\)`") "\\1" desc in
Some (subcmd_name, desc, { entries; subcommands = []; positionals = []; description = desc })
) sections
let read_manpage_file path =
if Filename.check_suffix path ".gz" then begin
let ic = Gzip.open_in path in
let buf = Buffer.create 8192 in
let chunk = Bytes.create 8192 in
(try while true do
let n = Gzip.input ic chunk 0 8192 in
if n = 0 then raise Exit
else Buffer.add_subbytes buf chunk 0 n
done with Exit | End_of_file -> ());
Gzip.close_in ic;
Buffer.contents buf
end else begin
let ic = open_in path in
let n = in_channel_length ic in
let s = Bytes.create n in
really_input ic s 0 n;
close_in ic;
Bytes.to_string s
end
let parse_manpage_file path =
read_manpage_file path |> parse_manpage_string

163
lib/nushell.ml Normal file
View file

@ -0,0 +1,163 @@
open Parser
module SSet = Set.Make(String)
module SMap = Map.Make(String)
module CSet = Set.Make(Char)
(* Nushell built-in commands and keywords *)
let nushell_builtins = [
"alias"; "all"; "ansi"; "any"; "append"; "ast"; "attr";
"bits"; "break"; "bytes";
"cal"; "cd"; "char"; "chunk-by"; "chunks"; "clear"; "collect";
"columns"; "commandline"; "compact"; "complete"; "config"; "const";
"continue"; "cp";
"date"; "debug"; "decode"; "def"; "default"; "describe"; "detect";
"do"; "drop"; "du";
"each"; "echo"; "encode"; "enumerate"; "error"; "every"; "exec";
"exit"; "explain"; "explore"; "export"; "export-env"; "extern";
"fill"; "filter"; "find"; "first"; "flatten"; "for"; "format"; "from";
"generate"; "get"; "glob"; "grid"; "group-by";
"hash"; "headers"; "help"; "hide"; "hide-env"; "histogram";
"history"; "http";
"if"; "ignore"; "input"; "insert"; "inspect"; "interleave"; "into";
"is-admin"; "is-empty"; "is-not-empty"; "is-terminal"; "items";
"job"; "join";
"keybindings"; "kill";
"last"; "length"; "let"; "let-env"; "lines"; "load-env"; "loop"; "ls";
"match"; "math"; "merge"; "metadata"; "mkdir"; "mktemp"; "module";
"move"; "mut"; "mv";
"nu-check"; "nu-highlight";
"open"; "overlay";
"panic"; "par-each"; "parse"; "path"; "plugin"; "port"; "prepend"; "print"; "ps";
"query";
"random"; "reduce"; "reject"; "rename"; "return"; "reverse"; "rm";
"roll"; "rotate"; "run-external";
"save"; "schema"; "scope"; "select"; "seq"; "shuffle"; "skip"; "sleep";
"slice"; "sort"; "sort-by"; "source"; "source-env"; "split"; "start";
"stor"; "str"; "sys";
"table"; "take"; "tee"; "term"; "timeit"; "to"; "touch"; "transpose";
"try"; "tutor";
"ulimit"; "umask"; "uname"; "uniq"; "uniq-by"; "unlet"; "update";
"upsert"; "url"; "use";
"values"; "version"; "view";
"watch"; "where"; "which"; "while"; "whoami"; "window"; "with-env"; "wrap";
"zip";
]
let builtin_set = lazy (SSet.of_list nushell_builtins)
let is_nushell_builtin cmd =
SSet.mem cmd (Lazy.force builtin_set)
let dedup_entries entries =
let key_of entry =
match entry.switch with
| Short c -> Printf.sprintf "-%c" c
| Long l | Both (_, l) -> Printf.sprintf "--%s" l
in
let score entry =
let sw = match entry.switch with Both _ -> 10 | _ -> 0 in
let p = match entry.param with Some _ -> 5 | None -> 0 in
let d = min 5 (String.length entry.desc / 10) in
sw + p + d
in
let best = List.fold_left (fun acc e ->
let k = key_of e in
match SMap.find_opt k acc with
| Some prev when score prev >= score e -> acc
| _ -> SMap.add k e acc
) SMap.empty entries in
let covered = SMap.fold (fun _ e acc ->
match e.switch with
| Both (c, _) -> CSet.add c acc
| _ -> acc
) best CSet.empty in
List.fold_left (fun (seen, acc) e ->
let k = key_of e in
if SSet.mem k seen then (seen, acc)
else match e.switch with
| Short c when CSet.mem c covered -> (seen, acc)
| _ -> (SSet.add k seen, SMap.find k best :: acc)
) (SSet.empty, []) entries |> snd |> List.rev
let nushell_type_of_param = function
| "FILE" | "file" | "PATH" | "path" | "DIR" | "dir" | "DIRECTORY"
| "FILENAME" | "PATTERNFILE" -> "path"
| "NUM" | "N" | "COUNT" | "NUMBER" | "int" | "INT" | "COLS" | "WIDTH"
| "LINES" | "DEPTH" | "depth" -> "int"
| _ -> "string"
let escape_nu s =
if not (String.contains s '"') && not (String.contains s '\\') then s
else begin
let buf = Buffer.create (String.length s + 4) in
String.iter (fun c -> match c with
| '"' -> Buffer.add_string buf "\\\""
| '\\' -> Buffer.add_string buf "\\\\"
| _ -> Buffer.add_char buf c
) s;
Buffer.contents buf
end
let format_flag entry =
let name = match entry.switch with
| Both (s, l) -> Printf.sprintf "--%s(-%c)" l s
| Long l -> Printf.sprintf "--%s" l
| Short s -> Printf.sprintf "-%c" s
in
let typed = match entry.param with
| Some (Mandatory p) | Some (Optional p) -> ": " ^ nushell_type_of_param p
| None -> ""
in
let flag = " " ^ name ^ typed in
if String.length entry.desc = 0 then flag
else
let pad_len = max 1 (40 - String.length flag) in
flag ^ String.make pad_len ' ' ^ "# " ^ entry.desc
let format_positional p =
let name = String.map (function '-' -> '_' | c -> c) p.pos_name in
let prefix = if p.variadic then "..." else "" in
let suffix = if p.optional && not p.variadic then "?" else "" in
let typ = nushell_type_of_param (String.uppercase_ascii p.pos_name) in
Printf.sprintf " %s%s%s: %s" prefix name suffix typ
let fixup_positionals positionals =
(* Nushell rules: no required after optional, only one rest param *)
List.fold_left (fun (saw_opt, saw_rest, acc) p ->
if p.variadic then
if saw_rest then (saw_opt, saw_rest, acc)
else (true, true, p :: acc)
else if saw_opt then
(true, saw_rest, { p with optional = true } :: acc)
else
(p.optional, saw_rest, p :: acc)
) (false, false, []) positionals
|> fun (_, _, acc) -> List.rev acc
let extern_of cmd_name result =
let entries = dedup_entries result.entries in
let cmd = escape_nu cmd_name in
let positionals = fixup_positionals result.positionals in
let pos_lines = List.map (fun p -> format_positional p ^ "\n") positionals in
let flags = List.map (fun e -> format_flag e ^ "\n") entries in
let main = Printf.sprintf "export extern \"%s\" [\n%s%s]\n" cmd (String.concat "" pos_lines) (String.concat "" flags) in
let subs = List.map (fun (sc : subcommand) ->
Printf.sprintf "\nexport extern \"%s %s\" [ # %s\n]\n"
cmd (escape_nu sc.name) (escape_nu sc.desc)
) result.subcommands in
String.concat "" (main :: subs)
let generate_extern = extern_of
let module_name_of cmd_name =
let s = String.map (function
| ('a'..'z' | 'A'..'Z' | '0'..'9' | '-' | '_') as c -> c | _ -> '-') cmd_name in
s ^ "-completions"
let generate_module cmd_name result =
let m = module_name_of cmd_name in
Printf.sprintf "module %s {\n%s}\n\nuse %s *\n" m (extern_of cmd_name result) m
let generate_extern_from_entries cmd_name entries =
generate_extern cmd_name { entries; subcommands = []; positionals = []; description = "" }

587
lib/parser.ml Normal file
View file

@ -0,0 +1,587 @@
open Angstrom
(* Strip ANSI escape sequences and OSC hyperlinks from --help output *)
let strip_ansi s =
let buf = Buffer.create (String.length s) in
let len = String.length s in
let i = ref 0 in
while !i < len do
if !i + 1 < len && Char.code s.[!i] = 0x1b then begin
let next = s.[!i + 1] in
if next = '[' then begin
(* CSI sequence: ESC [ ... final_byte *)
i := !i + 2;
while !i < len && not (s.[!i] >= '@' && s.[!i] <= '~') do incr i done;
if !i < len then incr i
end else if next = ']' then begin
(* OSC sequence: ESC ] ... (terminated by BEL or ESC \) *)
i := !i + 2;
let found = ref false in
while !i < len && not !found do
if s.[!i] = '\x07' then
(incr i; found := true)
else if !i + 1 < len && Char.code s.[!i] = 0x1b && s.[!i + 1] = '\\' then
(i := !i + 2; found := true)
else
incr i
done
end else begin
(* Other ESC sequence, skip ESC + one char *)
i := !i + 2
end
end else begin
Buffer.add_char buf s.[!i];
incr i
end
done;
Buffer.contents buf
let is_whitespace = function ' ' | '\t' -> true | _ -> false
let is_alphanumeric = function
| 'A' .. 'Z' | 'a' .. 'z' | '0' .. '9' -> true
| _ -> false
let is_param_char = function
| 'A' .. 'Z' | 'a' .. 'z' | '0' .. '9' | '_' | '-' -> true
| _ -> false
let is_upper_or_underscore = function
| 'A' .. 'Z' | '_' -> true
| _ -> false
let is_long_char = function
| 'A' .. 'Z' | 'a' .. 'z' | '0' .. '9' | '-' -> true
| _ -> false
type switch = Short of char | Long of string | Both of char * string
type param = Mandatory of string | Optional of string
type entry = { switch : switch; param : param option; desc : string }
type subcommand = { name : string; desc : string }
type positional = { pos_name : string; optional : bool; variadic : bool }
type help_result = { entries : entry list; subcommands : subcommand list; positionals : positional list; description : string }
(* --- Low-level combinators --- *)
let inline_ws = skip_while (function ' ' | '\t' -> true | _ -> false)
let eol = end_of_line <|> end_of_input
let eol_strict = end_of_line (* Must consume a newline, no EOF match *)
let short_switch = char '-' *> satisfy is_alphanumeric
let long_switch = string "--" *> take_while1 is_long_char
let comma = char ',' *> inline_ws
(* Parameter parsers *)
let eq_opt_param =
string "[=" *> take_while1 is_param_char <* char ']' >>| fun a -> Optional a
let eq_man_param =
char '=' *> take_while1 is_param_char >>| fun a -> Mandatory a
(* Space-separated ALL_CAPS param: e.g. " FILE", " TIME_STYLE" *)
let space_upper_param =
char ' ' *> peek_char_fail >>= fun c ->
if is_upper_or_underscore c then
take_while1 is_param_char >>= fun name ->
(* Ensure it's truly all-uppercase (not a description word like "Do") *)
if String.length name >= 1 && String.for_all (fun c -> is_upper_or_underscore c || c >= '0' && c <= '9') name then
return (Mandatory name)
else
fail "not an all-caps param"
else
fail "not an uppercase param"
(* Angle-bracket param: e.g. "<file>", "<notation>" *)
let angle_param =
char '<' *> take_while1 (fun c -> c <> '>') <* char '>' >>| fun name ->
Mandatory name
(* Space + angle bracket param *)
let space_angle_param =
char ' ' *> angle_param
(* Optional angle bracket param: [<file>] *)
let opt_angle_param =
char '[' *> char '<' *> take_while1 (fun c -> c <> '>') <* char '>' <* char ']'
>>| fun name -> Optional name
let space_opt_angle_param =
char ' ' *> opt_angle_param
(* Go/Cobra style: space + lowercase type word like "string", "list", "int" *)
let space_type_param =
char ' ' *> peek_char_fail >>= fun c ->
if c >= 'a' && c <= 'z' then
take_while1 (fun c -> c >= 'a' && c <= 'z') >>= fun name ->
(* Only short type-like words *)
if String.length name <= 10 then
return (Mandatory name)
else
fail "too long for type param"
else
fail "not a lowercase type param"
let param_parser =
option None
(choice
[ eq_opt_param; eq_man_param;
space_opt_angle_param; space_angle_param;
space_upper_param; space_type_param ]
>>| fun a -> Some a)
(* Switch parser: -a, --all | -a --all | --all / -a | -a | --all *)
let switch_parser =
choice
[
(short_switch >>= fun s ->
comma *> long_switch >>| fun l -> Both (s, l));
(short_switch >>= fun s ->
char ' ' *> long_switch >>| fun l -> Both (s, l));
(long_switch >>= fun l ->
inline_ws *> char '/' *> inline_ws *>
short_switch >>| fun s -> Both (s, l));
(short_switch >>| fun s -> Short s);
(long_switch >>| fun l -> Long l);
]
(* --- Description parsing with multi-line continuation --- *)
(* Take the rest of the line as text (does not consume newline) *)
let rest_of_line = take_till (fun c -> c = '\n' || c = '\r')
(* Check if a line is a continuation line: deeply indented, doesn't start with '-' *)
let continuation_line =
peek_string 1 >>= fun _ ->
(* Must start with significant whitespace (8+ spaces or tab) *)
let count_indent s =
let n = ref 0 in
let i = ref 0 in
while !i < String.length s do
(match s.[!i] with
| ' ' -> incr n
| '\t' -> n := !n + 8
| _ -> i := String.length s);
incr i
done;
!n
in
available >>= fun avail ->
if avail = 0 then fail "eof"
else
(* Peek ahead to see indentation level *)
peek_string (min avail 80) >>= fun preview ->
let indent = count_indent preview in
let trimmed = String.trim preview in
let starts_with_dash =
String.length trimmed > 0 && trimmed.[0] = '-'
in
if indent >= 8 && not starts_with_dash then
(* This is a continuation line — consume whitespace + text *)
inline_ws *> rest_of_line <* eol
else
fail "not a continuation line"
let description =
inline_ws *> rest_of_line <* eol >>= fun first_line ->
many continuation_line >>| fun cont_lines ->
let all = first_line :: cont_lines in
let all = List.filter (fun s -> String.length (String.trim s) > 0) all in
String.concat " " (List.map String.trim all)
(* Description that appears on a separate line below the flag (Clap long style) *)
let description_below =
many1 continuation_line >>| fun lines ->
let lines = List.filter (fun s -> String.length (String.trim s) > 0) lines in
String.concat " " (List.map String.trim lines)
(* --- Line classification for skipping --- *)
(* An option line starts with whitespace then '-' *)
let at_option_line =
peek_string 1 >>= fun _ ->
available >>= fun avail ->
if avail = 0 then fail "eof"
else
peek_string (min avail 40) >>= fun preview ->
let s = String.trim preview in
if String.length s > 0 && s.[0] = '-' then return ()
else fail "not an option line"
(* Skip a non-option line (section header, blank, description-only, etc.) *)
let skip_non_option_line =
(* Don't skip if this looks like an option line *)
(at_option_line *> fail "this is an option line")
<|> (rest_of_line *> eol_strict *> return ())
(* --- Entry parsing --- *)
(* Parse a single flag entry *)
let entry =
inline_ws *>
lift2 (fun (sw, param) desc -> { switch = sw; param; desc })
(lift2 (fun a b -> (a, b)) switch_parser param_parser)
(description <|> (eol *> (description_below <|> return "")))
(* --- Subcommand parsing --- *)
(* A subcommand line: " name description"
Also handles argument placeholders: " name UNIT... description" *)
let is_subcommand_char = function
| 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '-' | '_' -> true
| _ -> false
(* Skip argument placeholders like UNIT..., [PATTERN...|PID...], <file>
that appear between the subcommand name and the description.
Only consumes single-space gaps — the two-space gap before the
description is left for the main parser. *)
let skip_arg_placeholders =
fix (fun self ->
(* Peek ahead: single space followed by arg-like token *)
available >>= fun avail ->
if avail < 2 then return ()
else
peek_string (min avail 2) >>= fun s2 ->
if String.length s2 >= 2 && s2.[0] = ' ' && s2.[1] <> ' ' then
(* Single space — could be an arg placeholder *)
let next = s2.[1] in
if next = '[' || next = '<'
|| (next >= 'A' && next <= 'Z') then
(* Peek the full token to check if it's ALL_CAPS/brackets *)
peek_string (min avail 80) >>= fun preview ->
(* Extract the token after the single space *)
let tok_start = 1 in
let tok_end = ref tok_start in
while !tok_end < String.length preview
&& preview.[!tok_end] <> ' '
&& preview.[!tok_end] <> '\n'
&& preview.[!tok_end] <> '\r' do
incr tok_end
done;
let tok = String.sub preview tok_start (!tok_end - tok_start) in
(* Accept as placeholder if it starts with [ or < or is ALL_CAPS
(possibly with dots, pipes, dashes) *)
let is_placeholder =
tok.[0] = '[' || tok.[0] = '<'
|| String.for_all (fun c ->
(c >= 'A' && c <= 'Z') || c = '_' || c = '-'
|| c = '.' || c = '|' || c = ',' || (c >= '0' && c <= '9')
) tok
in
if is_placeholder then
advance (1 + String.length tok) *> self
else return ()
else return ()
else return ())
let subcommand_entry =
inline_ws *>
take_while1 is_subcommand_char >>= fun name ->
if String.length name < 2 then fail "subcommand name too short"
else
skip_arg_placeholders *>
char ' ' *> char ' ' *> inline_ws *>
rest_of_line <* eol >>| fun desc ->
{ name = String.lowercase_ascii name;
desc = let t = String.trim desc in
if String.length t >= 2 && t.[0] = '-' && t.[1] = ' ' then
String.trim (String.sub t 2 (String.length t - 2))
else t }
(* --- Section header detection --- *)
(* Detect lines like "Arguments:", "POSITIONALS:", etc. that introduce
positional-argument sections (where name+desc lines are NOT subcommands) *)
let is_arg_section s =
let lc = String.lowercase_ascii (String.trim s) in
let base = if String.ends_with ~suffix:":" lc
then String.sub lc 0 (String.length lc - 1) |> String.trim
else lc in
base = "arguments" || base = "args" || base = "positionals"
|| base = "positional arguments"
(* A section header: left-aligned (or lightly indented) text ending with ':',
not starting with '-'. Must be consumed BEFORE subcommand_entry in choice. *)
let section_header =
available >>= fun avail ->
if avail = 0 then fail "eof"
else
peek_string (min avail 80) >>= fun preview ->
(* Extract just the first line from the preview *)
let first_line = match String.index_opt preview '\n' with
| Some i -> String.sub preview 0 i
| None -> preview in
let t = String.trim first_line in
let len = String.length t in
let indent = let i = ref 0 in
while !i < String.length first_line && (first_line.[!i] = ' ' || first_line.[!i] = '\t') do incr i done;
!i in
if len >= 2 && t.[len - 1] = ':' && t.[0] <> '-' && indent <= 4 then
rest_of_line <* eol_strict >>| fun line -> is_arg_section line
else fail "not a section header"
(* --- Top-level parser --- *)
(* The main help parser: walks through lines, skipping non-option content,
collecting entries and subcommands *)
let help_parser =
let open Angstrom in
fix (fun _self ->
(* Try to parse an entry *)
let try_entry =
entry >>| fun e -> `Entry e
in
(* Detect section headers to track arg vs command sections *)
let try_section =
section_header >>| fun is_arg -> `Section is_arg
in
(* Try to parse a subcommand *)
let try_subcommand =
subcommand_entry >>| fun sc -> `Subcommand sc
in
(* Skip one non-option line *)
let try_skip =
skip_non_option_line >>| fun () -> `Skip
in
many (choice [ try_entry; try_section; try_subcommand; try_skip ]) >>| fun items ->
let entries = List.filter_map (function `Entry e -> Some e | _ -> None) items in
(* Only keep subcommands that didn't appear under an Arguments/Positionals section *)
let subcommands =
List.fold_left (fun (in_arg_sec, acc) item ->
match item with
| `Section is_arg -> (is_arg, acc)
| `Subcommand sc when not in_arg_sec -> (in_arg_sec, sc :: acc)
| _ -> (in_arg_sec, acc)
) (false, []) items
|> snd |> List.rev
|> List.fold_left (fun acc sc ->
match List.assoc_opt sc.name acc with
| Some prev when String.length prev.desc >= String.length sc.desc -> acc
| _ -> (sc.name, sc) :: List.remove_assoc sc.name acc
) []
|> List.rev_map snd
in
{ entries; subcommands; positionals = []; description = "" })
let skip_command_prefix s =
let len = String.length s in
let i = ref 0 in
let skip_ws () = while !i < len && (s.[!i] = ' ' || s.[!i] = '\t') do incr i done in
let is_word_char = function
| 'a'..'z' | 'A'..'Z' | '0'..'9' | '-' | '_' | '/' | '.' -> true
| _ -> false
in
let rec loop () =
skip_ws ();
if !i >= len then ()
else if s.[!i] = '[' || s.[!i] = '<' || s.[!i] = '(' || s.[!i] = '{' || s.[!i] = '-' then ()
else if is_word_char s.[!i] then begin
let start = !i in
while !i < len && is_word_char s.[!i] do incr i done;
let word = String.sub s start (!i - start) in
let has_lower = ref false in
String.iter (fun c -> if c >= 'a' && c <= 'z' then has_lower := true) word;
if not !has_lower then
i := start
else
loop ()
end
in
loop ();
!i
let parse_usage_args s =
let len = String.length s in
let i = ref 0 in
let results = ref [] in
let skip_ws () =
while !i < len && (s.[!i] = ' ' || s.[!i] = '\t') do incr i done in
let is_pos_char c =
(c >= 'A' && c <= 'Z') || c = '_' || c = '-' || (c >= '0' && c <= '9') in
let read_dots () =
skip_ws ();
if !i + 2 < len && s.[!i] = '.' && s.[!i+1] = '.' && s.[!i+2] = '.' then
(i := !i + 3; true)
else if !i + 2 < len && s.[!i] = '\xe2' && s.[!i+1] = '\x80' && s.[!i+2] = '\xa6' then
(i := !i + 3; true) (* UTF-8 ellipsis … *)
else false
in
let is_skip name =
let u = String.uppercase_ascii name in
u = "OPTIONS" || u = "OPTION" || u = "FLAGS" || u = "FLAG"
in
let is_clean_name name =
String.length name >= 2
&& String.for_all (fun c ->
(c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
|| (c >= '0' && c <= '9') || c = '_' || c = '-') name
in
let is_letter c = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') in
let skip_braces () =
(* Skip {A|c|d|...} alternative blocks *)
if !i < len && s.[!i] = '{' then begin
let depth = ref 1 in
incr i;
while !i < len && !depth > 0 do
if s.[!i] = '{' then incr depth
else if s.[!i] = '}' then decr depth;
incr i
done;
ignore (read_dots ());
true
end else false
in
while !i < len do
skip_ws ();
if !i >= len then ()
else if skip_braces () then ()
else match s.[!i] with
| '[' ->
incr i;
let start = !i in
let depth = ref 1 in
while !i < len && !depth > 0 do
if s.[!i] = '[' then incr depth
else if s.[!i] = ']' then decr depth;
incr i
done;
let bracket_end = !i - 1 in
let inner = String.sub s start (max 0 (bracket_end - start)) |> String.trim in
let inner, has_inner_dots =
if String.ends_with ~suffix:"..." inner then
(String.sub inner 0 (String.length inner - 3) |> String.trim, true)
else (inner, false)
in
let variadic = has_inner_dots || read_dots () in
if String.length inner > 0
&& inner.[0] <> '-'
&& (is_letter inner.[0] || inner.[0] = '<') then begin
let name =
if inner.[0] = '<' then
let e = try String.index inner '>' with Not_found -> String.length inner in
String.sub inner 1 (e - 1)
else inner
in
if is_clean_name name && not (is_skip name) then
results := { pos_name = String.lowercase_ascii name;
optional = true; variadic } :: !results
end
| '<' ->
incr i;
let start = !i in
while !i < len && s.[!i] <> '>' do incr i done;
let name = String.sub s start (!i - start) in
if !i < len then incr i;
let variadic = read_dots () in
if is_clean_name name && not (is_skip name) then
results := { pos_name = String.lowercase_ascii name;
optional = false; variadic } :: !results
| '-' ->
while !i < len && s.[!i] <> ' ' && s.[!i] <> '\t' && s.[!i] <> ']' do incr i done
| c when c >= 'A' && c <= 'Z' ->
let start = !i in
while !i < len && is_pos_char s.[!i] do incr i done;
let name = String.sub s start (!i - start) in
let variadic = read_dots () in
if String.length name >= 2
&& String.for_all (fun c ->
(c >= 'A' && c <= 'Z') || c = '_' || c = '-' || (c >= '0' && c <= '9')
) name
&& not (is_skip name) then
results := { pos_name = String.lowercase_ascii name;
optional = false; variadic } :: !results
| _ ->
incr i
done;
List.rev !results
|> List.fold_left (fun (seen, acc) p ->
if List.mem p.pos_name seen then (seen, acc)
else (p.pos_name :: seen, p :: acc)
) ([], [])
|> snd |> List.rev
let extract_usage_positionals text =
let lines = String.split_on_char '\n' text in
let lines_arr = Array.of_list lines in
let len = Array.length lines_arr in
let find_usage_line () =
let rec go i =
if i >= len then None
else
let t = String.trim lines_arr.(i) in
let tlen = String.length t in
let lc = String.lowercase_ascii t in
if tlen >= 6 && String.sub lc 0 6 = "usage:" then begin
let after = String.sub t 6 (tlen - 6) |> String.trim in
if String.length after > 0 then Some after
else if i + 1 < len then
(* Clap style: USAGE:\n cmd [OPTIONS] PATTERN *)
let next = String.trim lines_arr.(i + 1) in
if String.length next > 0 then Some next else None
else None
end else if lc = "usage" then begin
if i + 1 < len then
let next = String.trim lines_arr.(i + 1) in
if String.length next > 0 then Some next else None
else None
end else go (i + 1)
in
go 0
in
match find_usage_line () with
| None -> []
| Some usage ->
let cmd_end = skip_command_prefix usage in
let args = String.sub usage cmd_end (String.length usage - cmd_end) in
parse_usage_args args
let extract_cli11_positionals text =
let lines = String.split_on_char '\n' text in
let rec find_section = function
| [] -> []
| line :: rest ->
let t = String.trim line in
if t = "POSITIONALS:" || t = "Positionals:" then
parse_lines rest []
else
find_section rest
and parse_lines lines acc =
match lines with
| [] -> List.rev acc
| line :: rest ->
let len = String.length line in
if len = 0 || (line.[0] <> ' ' && line.[0] <> '\t') then
List.rev acc
else
let t = String.trim line in
if String.length t = 0 then List.rev acc
else match parse_one t with
| Some p -> parse_lines rest (p :: acc)
| None -> parse_lines rest acc
and parse_one s =
let len = String.length s in
let i = ref 0 in
let is_name_char c =
(c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|| (c >= '0' && c <= '9') || c = '_' || c = '-' in
while !i < len && is_name_char s.[!i] do incr i done;
if !i < 2 then None
else
let name = String.sub s 0 !i in
while !i < len && (s.[!i] = ' ' || s.[!i] = '\t') do incr i done;
(* skip type word: TEXT, INT, FLOAT, ENUM, BOOLEAN, etc. *)
while !i < len && s.[!i] >= 'A' && s.[!i] <= 'Z' do incr i done;
while !i < len && (s.[!i] = ' ' || s.[!i] = '\t') do incr i done;
let variadic = !i + 2 < len && s.[!i] = '.' && s.[!i+1] = '.' && s.[!i+2] = '.' in
Some { pos_name = String.lowercase_ascii name; optional = false; variadic }
in
find_section lines
let parse_help txt =
let clean = strip_ansi txt in
match Angstrom.parse_string ~consume:Consume.Prefix help_parser clean with
| Ok result ->
let cli11 = extract_cli11_positionals clean in
let usage = extract_usage_positionals clean in
let positionals = if cli11 <> [] then cli11 else usage in
Ok { result with positionals }
| Error msg -> Error msg

366
lib/store.ml Normal file
View file

@ -0,0 +1,366 @@
open Parser
let default_store_path () =
let cache = try Sys.getenv "XDG_CACHE_HOME"
with Not_found -> Filename.concat (Sys.getenv "HOME") ".cache" in
Filename.concat cache "inshellah"
let ensure_dir dir =
let rec mkdir_p d =
if Sys.file_exists d then ()
else begin mkdir_p (Filename.dirname d); Unix.mkdir d 0o755 end in
mkdir_p dir
let filename_of_command cmd =
String.map (function
| ' ' -> '_'
| ('a'..'z' | 'A'..'Z' | '0'..'9' | '-' | '_' | '.') as c -> c
| _ -> '-') cmd
let command_of_filename base =
String.map (function '_' -> ' ' | c -> c) base
(* --- JSON serialization of help_result --- *)
let escape_json s =
let buf = Buffer.create (String.length s + 4) in
String.iter (fun c -> match c with
| '"' -> Buffer.add_string buf "\\\""
| '\\' -> Buffer.add_string buf "\\\\"
| '\n' -> Buffer.add_string buf "\\n"
| '\t' -> Buffer.add_string buf "\\t"
| '\r' -> Buffer.add_string buf "\\r"
| c when Char.code c < 0x20 ->
Buffer.add_string buf (Printf.sprintf "\\u%04x" (Char.code c))
| c -> Buffer.add_char buf c
) s;
Buffer.contents buf
let json_string s = Printf.sprintf "\"%s\"" (escape_json s)
let json_null = "null"
let json_switch_of = function
| Short c -> Printf.sprintf "{\"type\":\"short\",\"char\":%s}" (json_string (String.make 1 c))
| Long l -> Printf.sprintf "{\"type\":\"long\",\"name\":%s}" (json_string l)
| Both (c, l) ->
Printf.sprintf "{\"type\":\"both\",\"char\":%s,\"name\":%s}"
(json_string (String.make 1 c)) (json_string l)
let json_param_of = function
| None -> json_null
| Some (Mandatory p) ->
Printf.sprintf "{\"kind\":\"mandatory\",\"name\":%s}" (json_string p)
| Some (Optional p) ->
Printf.sprintf "{\"kind\":\"optional\",\"name\":%s}" (json_string p)
let json_entry_of e =
Printf.sprintf "{\"switch\":%s,\"param\":%s,\"desc\":%s}"
(json_switch_of e.switch) (json_param_of e.param) (json_string e.desc)
let json_subcommand_of sc =
Printf.sprintf "{\"name\":%s,\"desc\":%s}" (json_string sc.name) (json_string sc.desc)
let json_positional_of p =
Printf.sprintf "{\"name\":%s,\"optional\":%b,\"variadic\":%b}"
(json_string p.pos_name) p.optional p.variadic
let json_list f items =
"[" ^ String.concat "," (List.map f items) ^ "]"
let json_of_help_result ?(source="help") r =
Printf.sprintf "{\"source\":%s,\"description\":%s,\"entries\":%s,\"subcommands\":%s,\"positionals\":%s}"
(json_string source)
(json_string r.description)
(json_list json_entry_of r.entries)
(json_list json_subcommand_of r.subcommands)
(json_list json_positional_of r.positionals)
(* --- JSON deserialization --- *)
(* Minimal JSON parser — just enough for our own output *)
type json =
| Jnull
| Jbool of bool
| Jstring of string
| Jarray of json list
| Jobject of (string * json) list
let json_get key = function
| Jobject pairs -> (try List.assoc key pairs with Not_found -> Jnull)
| _ -> Jnull
let json_to_string = function Jstring s -> s | _ -> ""
let json_to_bool = function Jbool b -> b | _ -> false
let json_to_list = function Jarray l -> l | _ -> []
exception Json_error of string
let parse_json s =
let len = String.length s in
let pos = ref 0 in
let peek () = if !pos < len then s.[!pos] else '\x00' in
let advance () = incr pos in
let skip_ws () =
while !pos < len && (s.[!pos] = ' ' || s.[!pos] = '\t'
|| s.[!pos] = '\n' || s.[!pos] = '\r') do
advance ()
done in
let expect c =
skip_ws ();
if peek () <> c then
raise (Json_error (Printf.sprintf "expected '%c' at %d" c !pos));
advance () in
let rec parse_value () =
skip_ws ();
match peek () with
| '"' -> Jstring (parse_string ())
| '{' -> parse_object ()
| '[' -> parse_array ()
| 'n' -> advance (); advance (); advance (); advance (); Jnull
| 't' -> advance (); advance (); advance (); advance (); Jbool true
| 'f' ->
advance (); advance (); advance (); advance (); advance (); Jbool false
| c -> raise (Json_error (Printf.sprintf "unexpected '%c' at %d" c !pos))
and parse_string () =
expect '"';
let buf = Buffer.create 32 in
while peek () <> '"' do
if peek () = '\\' then begin
advance ();
(match peek () with
| '"' -> Buffer.add_char buf '"'
| '\\' -> Buffer.add_char buf '\\'
| 'n' -> Buffer.add_char buf '\n'
| 't' -> Buffer.add_char buf '\t'
| 'r' -> Buffer.add_char buf '\r'
| 'u' ->
advance ();
let hex = String.sub s !pos 4 in
pos := !pos + 3;
let code = int_of_string ("0x" ^ hex) in
if code < 128 then Buffer.add_char buf (Char.chr code)
else begin
(* UTF-8 encode *)
if code < 0x800 then begin
Buffer.add_char buf (Char.chr (0xc0 lor (code lsr 6)));
Buffer.add_char buf (Char.chr (0x80 lor (code land 0x3f)))
end else begin
Buffer.add_char buf (Char.chr (0xe0 lor (code lsr 12)));
Buffer.add_char buf (Char.chr (0x80 lor ((code lsr 6) land 0x3f)));
Buffer.add_char buf (Char.chr (0x80 lor (code land 0x3f)))
end
end
| c -> Buffer.add_char buf c);
advance ()
end else begin
Buffer.add_char buf (peek ());
advance ()
end
done;
advance (); (* closing quote *)
Buffer.contents buf
and parse_object () =
expect '{';
skip_ws ();
if peek () = '}' then (advance (); Jobject [])
else begin
let pairs = ref [] in
let cont = ref true in
while !cont do
skip_ws ();
let key = parse_string () in
expect ':';
let value = parse_value () in
pairs := (key, value) :: !pairs;
skip_ws ();
if peek () = ',' then advance ()
else cont := false
done;
expect '}';
Jobject (List.rev !pairs)
end
and parse_array () =
expect '[';
skip_ws ();
if peek () = ']' then (advance (); Jarray [])
else begin
let items = ref [] in
let cont = ref true in
while !cont do
let v = parse_value () in
items := v :: !items;
skip_ws ();
if peek () = ',' then advance ()
else cont := false
done;
expect ']';
Jarray (List.rev !items)
end
in
parse_value ()
let switch_of_json j =
match json_to_string (json_get "type" j) with
| "short" ->
let c = json_to_string (json_get "char" j) in
Short (if String.length c > 0 then c.[0] else '?')
| "long" -> Long (json_to_string (json_get "name" j))
| "both" ->
let c = json_to_string (json_get "char" j) in
Both ((if String.length c > 0 then c.[0] else '?'),
json_to_string (json_get "name" j))
| _ -> Long "?"
let param_of_json = function
| Jnull -> None
| j ->
let name = json_to_string (json_get "name" j) in
(match json_to_string (json_get "kind" j) with
| "mandatory" -> Some (Mandatory name)
| "optional" -> Some (Optional name)
| _ -> None)
let entry_of_json j =
{ switch = switch_of_json (json_get "switch" j);
param = param_of_json (json_get "param" j);
desc = json_to_string (json_get "desc" j) }
let subcommand_of_json j =
{ name = json_to_string (json_get "name" j);
desc = json_to_string (json_get "desc" j) }
let positional_of_json j =
{ pos_name = json_to_string (json_get "name" j);
optional = json_to_bool (json_get "optional" j);
variadic = json_to_bool (json_get "variadic" j) }
let help_result_of_json j =
{ entries = List.map entry_of_json (json_to_list (json_get "entries" j));
subcommands = List.map subcommand_of_json (json_to_list (json_get "subcommands" j));
positionals = List.map positional_of_json (json_to_list (json_get "positionals" j));
description = json_to_string (json_get "description" j) }
(* --- Filesystem operations --- *)
let write_file path contents =
let oc = open_out path in
output_string oc contents;
close_out oc
let read_file path =
try
let ic = open_in path in
let n = in_channel_length ic in
let s = Bytes.create n in
really_input ic s 0 n;
close_in ic;
Some (Bytes.to_string s)
with _ -> None
let write_result ~dir ?(source="help") command result =
let path = Filename.concat dir (filename_of_command command ^ ".json") in
write_file path (json_of_help_result ~source result)
let write_native ~dir command data =
let path = Filename.concat dir (filename_of_command command ^ ".nu") in
write_file path data
let is_dir path = Sys.file_exists path && Sys.is_directory path
let find_file dirs command =
let base = filename_of_command command in
List.find_map (fun dir ->
let json_path = Filename.concat dir (base ^ ".json") in
if Sys.file_exists json_path then Some json_path
else
let nu_path = Filename.concat dir (base ^ ".nu") in
if Sys.file_exists nu_path then Some nu_path
else None
) dirs
let lookup dirs command =
let base = filename_of_command command in
List.find_map (fun dir ->
let path = Filename.concat dir (base ^ ".json") in
match read_file path with
| Some data ->
(try Some (help_result_of_json (parse_json data))
with _ -> None)
| None -> None
) dirs
let lookup_raw dirs command =
let base = filename_of_command command in
List.find_map (fun dir ->
let json_path = Filename.concat dir (base ^ ".json") in
match read_file json_path with
| Some _ as r -> r
| None ->
let nu_path = Filename.concat dir (base ^ ".nu") in
read_file nu_path
) dirs
let chop_extension f =
if Filename.check_suffix f ".json" then Some (Filename.chop_suffix f ".json")
else if Filename.check_suffix f ".nu" then Some (Filename.chop_suffix f ".nu")
else None
let subcommands_of dirs command =
let prefix = filename_of_command command ^ "_" in
let plen = String.length prefix in
let module SMap = Map.Make(String) in
let subs = List.fold_left (fun subs dir ->
if is_dir dir then
Array.fold_left (fun subs f ->
if not (String.starts_with ~prefix f) then subs
else
let is_json = Filename.check_suffix f ".json" in
match chop_extension f with
| None -> subs
| Some b ->
let rest = String.sub b plen (String.length b - plen) in
if String.contains rest '_' || String.length rest = 0 then subs
else if SMap.mem rest subs then subs
else
let desc = if is_json then
match read_file (Filename.concat dir f) with
| Some data ->
(try json_to_string (json_get "description" (parse_json data))
with _ -> "")
| None -> ""
else "" in
SMap.add rest { name = rest; desc } subs
) subs (Sys.readdir dir)
else subs
) SMap.empty dirs in
SMap.fold (fun _ sc acc -> sc :: acc) subs [] |> List.rev
let all_commands dirs =
let module SSet = Set.Make(String) in
List.fold_left (fun cmds dir ->
if is_dir dir then
Array.fold_left (fun cmds f ->
match chop_extension f with
| Some b -> SSet.add (command_of_filename b) cmds
| None -> cmds
) cmds (Sys.readdir dir)
else cmds
) SSet.empty dirs
|> SSet.elements
let file_type_of dirs command =
let base = filename_of_command command in
List.find_map (fun dir ->
let json_path = Filename.concat dir (base ^ ".json") in
if Sys.file_exists json_path then
(match read_file json_path with
| Some data ->
(try Some (json_to_string (json_get "source" (parse_json data)))
with _ -> Some "json")
| None -> Some "json")
else
let nu_path = Filename.concat dir (base ^ ".nu") in
if Sys.file_exists nu_path then Some "native"
else None
) dirs