From f2ed3286d0b3e81d70cbf2104691f59686beb5a7 Mon Sep 17 00:00:00 2001 From: atagen Date: Sun, 24 May 2026 18:30:05 +1000 Subject: [PATCH] parallelise fetches --- meat.nu | 113 ++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 86 insertions(+), 27 deletions(-) diff --git a/meat.nu b/meat.nu index 0fc1af9..533927f 100644 --- a/meat.nu +++ b/meat.nu @@ -70,6 +70,7 @@ def activate [build_path: string] { def do-build [extras: list = []] { let tmpdir = ^mktemp -d -t "meat-build.XXXXXX" | str trim let build = $"($tmpdir)/build" + warm-pins nix-build-nom $build $"($env.MEATS)/entry.nix" (system-attr) $extras differ-step $build activate $build @@ -90,6 +91,7 @@ def cmd-yum [...args: string] { def cmd-cook [...args: string] { with-frame { meat-print "PREPARING DELICIOUS MEATS.." + warm-pins try { ^nix-build --no-out-link $"($env.MEATS)/entry.nix" -A (system-attr) ...$args } } } @@ -97,6 +99,7 @@ def cmd-cook [...args: string] { def cmd-poke [...args: string] { with-frame { meat-print "PREPARING SUSPICIOUS MEATS.." + warm-pins try { ^nix-build --no-out-link --show-trace $"($env.MEATS)/entry.nix" -A (system-attr) ...$args } } } @@ -194,40 +197,82 @@ def prefetch-pin [url: string] { $j.locked | insert narHash $j.hash } +# Fetch one locked input into the store, mirroring lib/inputs.nix's +# `builtins.fetchTree lock.`. Returns the name on failure, else null. +def warm-pin [name: string, node: record] { + let tmp = ^mktemp -t "meat-pin.XXXXXX.json" | str trim + $node | to json | save -f $tmp + let expr = $"\(builtins.fetchTree \(builtins.fromJSON \(builtins.readFile \"($tmp)\"\)\)\).outPath" + let r = ^nix eval --impure --raw --expr $expr | complete + try { rm $tmp } + if $r.exit_code != 0 { $name } else { null } +} + +# Pre-fetch every locked input in parallel so the serial fetchTree calls +# during evaluation become cache hits. Warming is idempotent and non-fatal: +# anything that fails here is simply re-fetched by the build itself. +def warm-pins [] { + let lock = load-lock + let names = $lock | columns + if ($names | is-empty) { return } + meat-print "GATHERING MEATS.." + let failed = $names | par-each { |name| warm-pin $name ($lock | get $name) } | compact + for name in $failed { + meat-print $" COULDN'T GATHER ($name | str upcase) \(BUILD WILL RETRY\)" + } +} + def cmd-fresh [...names: string] { with-frame { meat-print "HUNTING FRESH MEATS.." let pins = load-pins let shorturls = $pins.shorturls? | default {} let inputs = $pins.inputs - mut lock = load-lock + let lock = load-lock - let targets = if ($names | is-empty) { $inputs | columns } else { $names } + let requested = if ($names | is-empty) { $inputs | columns } else { $names } - for name in $targets { + # Report unknown names up front and keep only real targets. + let targets = $requested | each { |name| if not ($name in ($inputs | columns)) { meat-print $"NO MEAT CALLED ($name | str upcase).." - continue - } - meat-print $"PROCESSING ($name | str upcase).." + null + } else { $name } + } | compact + + # Prefetch every target in parallel — this is the network-bound work. + # No lock writes happen here; results are collected and applied below. + let results = $targets | par-each { |name| + let old_rev = ($lock | get -o $name) | default {} | get -o rev try { let pin = $inputs | get $name let expanded = expand-shorturl $pin.url $shorturls let entry = prefetch-pin $expanded - let old_rev = ($lock | get -o $name) | default {} | get -o rev - let new_rev = $entry.rev - if $old_rev == $new_rev { - meat-print $" ($name | str upcase) STILL FRESH" - continue - } - $lock = $lock | upsert $name $entry - write-lock $lock - let from = if ($old_rev | is-empty) { "NEW" } else { $old_rev | str substring 0..8 } - meat-print $" ($name | str upcase): ($from) -> ($new_rev | str substring 0..8)" + { name: $name, ok: true, entry: $entry, old_rev: $old_rev, new_rev: $entry.rev } } catch { |e| - meat-print $" NO FIND ($name | str upcase): ($e.msg)" + { name: $name, ok: false, err: $e.msg } } } + + # Apply results sequentially in target order so the lock file is never + # written concurrently and reporting stays deterministic. + mut lock = $lock + for name in $targets { + let r = $results | where name == $name | first + meat-print $"PROCESSING ($name | str upcase).." + if not $r.ok { + meat-print $" NO FIND ($name | str upcase): ($r.err)" + continue + } + if $r.old_rev == $r.new_rev { + meat-print $" ($name | str upcase) STILL FRESH" + continue + } + $lock = $lock | upsert $name $r.entry + write-lock $lock + let from = if ($r.old_rev | is-empty) { "NEW" } else { $r.old_rev | str substring 0..8 } + meat-print $" ($name | str upcase): ($from) -> ($r.new_rev | str substring 0..8)" + } } print "" } @@ -240,21 +285,34 @@ def cmd-look [] { let inputs = $pins.inputs let lock = load-lock - let stale = $inputs | transpose name pin | each { |row| + let rows = $inputs | transpose name pin + + # Query every upstream head in parallel; no printing happens here. + let results = $rows | par-each { |row| + let old_rev = ($lock | get -o $row.name) | default {} | get -o rev try { let expanded = expand-shorturl $row.pin.url $shorturls let new_rev = ls-remote-head $expanded - let old_rev = ($lock | get -o $row.name) | default {} | get -o rev - if $old_rev != $new_rev { - let from = if ($old_rev | is-empty) { "NEW" } else { $old_rev | str substring 0..8 } - meat-print $" ($row.name | str upcase): ($from) -> ($new_rev | str substring 0..8)" - $row.name - } else { null } + { name: $row.name, ok: true, stale: ($old_rev != $new_rev), old_rev: $old_rev, new_rev: $new_rev } } catch { - meat-print $" NO FIND ($row.name | str upcase).." - null + { name: $row.name, ok: false } } - } | compact + } + + # Report in input order so output is deterministic and never races. + mut stale = [] + for row in $rows { + let r = $results | where name == $row.name | first + if not $r.ok { + meat-print $" NO FIND ($row.name | str upcase).." + continue + } + if $r.stale { + let from = if ($r.old_rev | is-empty) { "NEW" } else { $r.old_rev | str substring 0..8 } + meat-print $" ($row.name | str upcase): ($from) -> ($r.new_rev | str substring 0..8)" + $stale = ($stale | append $row.name) + } + } if ($stale | is-empty) { meat-print "NO MEAT FRESHER" } @@ -334,6 +392,7 @@ def cmd-ritual [] { let nix_conf = $"($tmpdir)/nix.conf" let build = $"($tmpdir)/build" try { + warm-pins nix-build-nom $nix_conf $"($meats)/entry.nix" (nix-conf-attr) meat-print "CONSUMING MEATS.." with-env { NIX_USER_CONF_FILES: $nix_conf } {