parallelise fetches
This commit is contained in:
atagen
parent
40fae13c23
commit
f2ed3286d0
1 changed files with 86 additions and 27 deletions
113
meat.nu
113
meat.nu
|
|
@ -70,6 +70,7 @@ def activate [build_path: string] {
|
|||
def do-build [extras: list<string> = []] {
|
||||
let tmpdir = ^mktemp -d -t "meat-build.XXXXXX" | str trim
|
||||
let build = $"($tmpdir)/build"
|
||||
warm-pins
|
||||
nix-build-nom $build $"($env.MEATS)/entry.nix" (system-attr) $extras
|
||||
differ-step $build
|
||||
activate $build
|
||||
|
|
@ -90,6 +91,7 @@ def cmd-yum [...args: string] {
|
|||
def cmd-cook [...args: string] {
|
||||
with-frame {
|
||||
meat-print "PREPARING DELICIOUS MEATS.."
|
||||
warm-pins
|
||||
try { ^nix-build --no-out-link $"($env.MEATS)/entry.nix" -A (system-attr) ...$args }
|
||||
}
|
||||
}
|
||||
|
|
@ -97,6 +99,7 @@ def cmd-cook [...args: string] {
|
|||
def cmd-poke [...args: string] {
|
||||
with-frame {
|
||||
meat-print "PREPARING SUSPICIOUS MEATS.."
|
||||
warm-pins
|
||||
try { ^nix-build --no-out-link --show-trace $"($env.MEATS)/entry.nix" -A (system-attr) ...$args }
|
||||
}
|
||||
}
|
||||
|
|
@ -194,40 +197,82 @@ def prefetch-pin [url: string] {
|
|||
$j.locked | insert narHash $j.hash
|
||||
}
|
||||
|
||||
# Fetch one locked input into the store, mirroring lib/inputs.nix's
|
||||
# `builtins.fetchTree lock.<name>`. Returns the name on failure, else null.
|
||||
def warm-pin [name: string, node: record] {
|
||||
let tmp = ^mktemp -t "meat-pin.XXXXXX.json" | str trim
|
||||
$node | to json | save -f $tmp
|
||||
let expr = $"\(builtins.fetchTree \(builtins.fromJSON \(builtins.readFile \"($tmp)\"\)\)\).outPath"
|
||||
let r = ^nix eval --impure --raw --expr $expr | complete
|
||||
try { rm $tmp }
|
||||
if $r.exit_code != 0 { $name } else { null }
|
||||
}
|
||||
|
||||
# Pre-fetch every locked input in parallel so the serial fetchTree calls
|
||||
# during evaluation become cache hits. Warming is idempotent and non-fatal:
|
||||
# anything that fails here is simply re-fetched by the build itself.
|
||||
def warm-pins [] {
|
||||
let lock = load-lock
|
||||
let names = $lock | columns
|
||||
if ($names | is-empty) { return }
|
||||
meat-print "GATHERING MEATS.."
|
||||
let failed = $names | par-each { |name| warm-pin $name ($lock | get $name) } | compact
|
||||
for name in $failed {
|
||||
meat-print $" COULDN'T GATHER ($name | str upcase) \(BUILD WILL RETRY\)"
|
||||
}
|
||||
}
|
||||
|
||||
def cmd-fresh [...names: string] {
|
||||
with-frame {
|
||||
meat-print "HUNTING FRESH MEATS.."
|
||||
let pins = load-pins
|
||||
let shorturls = $pins.shorturls? | default {}
|
||||
let inputs = $pins.inputs
|
||||
mut lock = load-lock
|
||||
let lock = load-lock
|
||||
|
||||
let targets = if ($names | is-empty) { $inputs | columns } else { $names }
|
||||
let requested = if ($names | is-empty) { $inputs | columns } else { $names }
|
||||
|
||||
for name in $targets {
|
||||
# Report unknown names up front and keep only real targets.
|
||||
let targets = $requested | each { |name|
|
||||
if not ($name in ($inputs | columns)) {
|
||||
meat-print $"NO MEAT CALLED ($name | str upcase).."
|
||||
continue
|
||||
}
|
||||
meat-print $"PROCESSING ($name | str upcase).."
|
||||
null
|
||||
} else { $name }
|
||||
} | compact
|
||||
|
||||
# Prefetch every target in parallel — this is the network-bound work.
|
||||
# No lock writes happen here; results are collected and applied below.
|
||||
let results = $targets | par-each { |name|
|
||||
let old_rev = ($lock | get -o $name) | default {} | get -o rev
|
||||
try {
|
||||
let pin = $inputs | get $name
|
||||
let expanded = expand-shorturl $pin.url $shorturls
|
||||
let entry = prefetch-pin $expanded
|
||||
let old_rev = ($lock | get -o $name) | default {} | get -o rev
|
||||
let new_rev = $entry.rev
|
||||
if $old_rev == $new_rev {
|
||||
meat-print $" ($name | str upcase) STILL FRESH"
|
||||
continue
|
||||
}
|
||||
$lock = $lock | upsert $name $entry
|
||||
write-lock $lock
|
||||
let from = if ($old_rev | is-empty) { "NEW" } else { $old_rev | str substring 0..8 }
|
||||
meat-print $" ($name | str upcase): ($from) -> ($new_rev | str substring 0..8)"
|
||||
{ name: $name, ok: true, entry: $entry, old_rev: $old_rev, new_rev: $entry.rev }
|
||||
} catch { |e|
|
||||
meat-print $" NO FIND ($name | str upcase): ($e.msg)"
|
||||
{ name: $name, ok: false, err: $e.msg }
|
||||
}
|
||||
}
|
||||
|
||||
# Apply results sequentially in target order so the lock file is never
|
||||
# written concurrently and reporting stays deterministic.
|
||||
mut lock = $lock
|
||||
for name in $targets {
|
||||
let r = $results | where name == $name | first
|
||||
meat-print $"PROCESSING ($name | str upcase).."
|
||||
if not $r.ok {
|
||||
meat-print $" NO FIND ($name | str upcase): ($r.err)"
|
||||
continue
|
||||
}
|
||||
if $r.old_rev == $r.new_rev {
|
||||
meat-print $" ($name | str upcase) STILL FRESH"
|
||||
continue
|
||||
}
|
||||
$lock = $lock | upsert $name $r.entry
|
||||
write-lock $lock
|
||||
let from = if ($r.old_rev | is-empty) { "NEW" } else { $r.old_rev | str substring 0..8 }
|
||||
meat-print $" ($name | str upcase): ($from) -> ($r.new_rev | str substring 0..8)"
|
||||
}
|
||||
}
|
||||
print ""
|
||||
}
|
||||
|
|
@ -240,21 +285,34 @@ def cmd-look [] {
|
|||
let inputs = $pins.inputs
|
||||
let lock = load-lock
|
||||
|
||||
let stale = $inputs | transpose name pin | each { |row|
|
||||
let rows = $inputs | transpose name pin
|
||||
|
||||
# Query every upstream head in parallel; no printing happens here.
|
||||
let results = $rows | par-each { |row|
|
||||
let old_rev = ($lock | get -o $row.name) | default {} | get -o rev
|
||||
try {
|
||||
let expanded = expand-shorturl $row.pin.url $shorturls
|
||||
let new_rev = ls-remote-head $expanded
|
||||
let old_rev = ($lock | get -o $row.name) | default {} | get -o rev
|
||||
if $old_rev != $new_rev {
|
||||
let from = if ($old_rev | is-empty) { "NEW" } else { $old_rev | str substring 0..8 }
|
||||
meat-print $" ($row.name | str upcase): ($from) -> ($new_rev | str substring 0..8)"
|
||||
$row.name
|
||||
} else { null }
|
||||
{ name: $row.name, ok: true, stale: ($old_rev != $new_rev), old_rev: $old_rev, new_rev: $new_rev }
|
||||
} catch {
|
||||
meat-print $" NO FIND ($row.name | str upcase).."
|
||||
null
|
||||
{ name: $row.name, ok: false }
|
||||
}
|
||||
} | compact
|
||||
}
|
||||
|
||||
# Report in input order so output is deterministic and never races.
|
||||
mut stale = []
|
||||
for row in $rows {
|
||||
let r = $results | where name == $row.name | first
|
||||
if not $r.ok {
|
||||
meat-print $" NO FIND ($row.name | str upcase).."
|
||||
continue
|
||||
}
|
||||
if $r.stale {
|
||||
let from = if ($r.old_rev | is-empty) { "NEW" } else { $r.old_rev | str substring 0..8 }
|
||||
meat-print $" ($row.name | str upcase): ($from) -> ($r.new_rev | str substring 0..8)"
|
||||
$stale = ($stale | append $row.name)
|
||||
}
|
||||
}
|
||||
if ($stale | is-empty) {
|
||||
meat-print "NO MEAT FRESHER"
|
||||
}
|
||||
|
|
@ -334,6 +392,7 @@ def cmd-ritual [] {
|
|||
let nix_conf = $"($tmpdir)/nix.conf"
|
||||
let build = $"($tmpdir)/build"
|
||||
try {
|
||||
warm-pins
|
||||
nix-build-nom $nix_conf $"($meats)/entry.nix" (nix-conf-attr)
|
||||
meat-print "CONSUMING MEATS.."
|
||||
with-env { NIX_USER_CONF_FILES: $nix_conf } {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue