init
This commit is contained in:
commit
fd80fbab7e
48 changed files with 16775 additions and 0 deletions
14
crates/jupiter-cache/Cargo.toml
Normal file
14
crates/jupiter-cache/Cargo.toml
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
[package]
|
||||
name = "jupiter-cache"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
jupiter-api-types = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
43
crates/jupiter-cache/src/error.rs
Normal file
43
crates/jupiter-cache/src/error.rs
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
//! Error types for the Nix binary cache.
|
||||
//!
|
||||
//! [`CacheError`] is the single error enum used across every layer of the
|
||||
//! cache -- storage I/O, NARInfo parsing, and capacity limits. Axum route
|
||||
//! handlers in [`crate::routes`] translate these variants into the appropriate
|
||||
//! HTTP status codes (404, 400, 500, etc.).
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// Unified error type for all binary cache operations.
|
||||
///
|
||||
/// Each variant maps to a different failure mode that can occur when reading,
|
||||
/// writing, or validating cache artefacts.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum CacheError {
|
||||
/// The requested store-path hash does not exist in the cache.
|
||||
///
|
||||
/// A store hash is the first 32 base-32 characters of a Nix store path
|
||||
/// (e.g. the `aaaa...` part of `/nix/store/aaaa...-hello-2.12`).
|
||||
/// This error is returned when neither a `.narinfo` file nor a
|
||||
/// corresponding NAR archive can be found on disk.
|
||||
#[error("store hash not found: {0}")]
|
||||
NotFound(String),
|
||||
|
||||
/// A low-level filesystem I/O error occurred while reading or writing
|
||||
/// cache data. This typically surfaces as an HTTP 500 to the client.
|
||||
#[error("IO error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
/// The NARInfo text failed to parse -- a required field is missing or a
|
||||
/// value is malformed. When this comes from a `PUT` request it results
|
||||
/// in an HTTP 400 (Bad Request) response.
|
||||
#[error("invalid narinfo: {0}")]
|
||||
InvalidNarInfo(String),
|
||||
|
||||
/// The on-disk cache has exceeded its configured maximum size.
|
||||
///
|
||||
/// This variant exists to support an optional size cap (`max_size_gb` in
|
||||
/// [`crate::store::LocalStore::new`]). When the cap is hit, further
|
||||
/// uploads are rejected until space is freed.
|
||||
#[error("storage full")]
|
||||
StorageFull,
|
||||
}
|
||||
44
crates/jupiter-cache/src/lib.rs
Normal file
44
crates/jupiter-cache/src/lib.rs
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
//! # jupiter-cache -- Built-in Nix Binary Cache for Jupiter
|
||||
//!
|
||||
//! Jupiter is a self-hosted, wire-compatible replacement for
|
||||
//! [hercules-ci.com](https://hercules-ci.com). This crate implements an
|
||||
//! **optional** built-in Nix binary cache server that speaks the same HTTP
|
||||
//! protocol used by `cache.nixos.org` and other standard Nix binary caches.
|
||||
//!
|
||||
//! ## Why an optional cache?
|
||||
//!
|
||||
//! In a typical Hercules CI deployment each agent already ships with its own
|
||||
//! binary cache support. However, some organisations prefer a single, shared
|
||||
//! cache server that every agent (and every developer workstation) can push to
|
||||
//! and pull from. `jupiter-cache` fills that role: it can be enabled inside
|
||||
//! the Jupiter server process so that no separate cache infrastructure (e.g.
|
||||
//! an S3 bucket or a dedicated `nix-serve` instance) is required.
|
||||
//!
|
||||
//! ## The Nix binary cache HTTP protocol
|
||||
//!
|
||||
//! The protocol is intentionally simple and fully compatible with the one
|
||||
//! implemented by `cache.nixos.org`:
|
||||
//!
|
||||
//! | Method | Path | Description |
|
||||
//! |--------|-------------------------------|-------------------------------------------|
|
||||
//! | `GET` | `/nix-cache-info` | Returns cache metadata (store dir, etc.) |
|
||||
//! | `GET` | `/{storeHash}.narinfo` | Returns the NARInfo for a store path hash |
|
||||
//! | `PUT` | `/{storeHash}.narinfo` | Uploads a NARInfo (agent -> cache) |
|
||||
//! | `GET` | `/nar/{narHash}.nar[.xz|.zst]`| Serves the actual NAR archive |
|
||||
//!
|
||||
//! A **NARInfo** file is the metadata envelope for a Nix store path. It
|
||||
//! describes the NAR archive's hash, compressed and uncompressed sizes,
|
||||
//! references to other store paths, the derivation that produced the path,
|
||||
//! and one or more cryptographic signatures that attest to its authenticity.
|
||||
//!
|
||||
//! ## Crate layout
|
||||
//!
|
||||
//! * [`error`] -- Error types for cache operations.
|
||||
//! * [`narinfo`] -- Parser and serialiser for the NARInfo text format.
|
||||
//! * [`store`] -- [`LocalStore`](store::LocalStore) -- on-disk storage backend.
|
||||
//! * [`routes`] -- Axum route handlers that expose the cache over HTTP.
|
||||
|
||||
pub mod error;
|
||||
pub mod narinfo;
|
||||
pub mod routes;
|
||||
pub mod store;
|
||||
233
crates/jupiter-cache/src/narinfo.rs
Normal file
233
crates/jupiter-cache/src/narinfo.rs
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
//! NARInfo parser and serialiser.
|
||||
//!
|
||||
//! Every path in a Nix binary cache is described by a **NARInfo** file. When
|
||||
//! a Nix client wants to know whether a particular store path is available in
|
||||
//! the cache it fetches `https://<cache>/<storeHash>.narinfo`. The response
|
||||
//! is a simple, line-oriented, key-value text format that looks like this:
|
||||
//!
|
||||
//! ```text
|
||||
//! StorePath: /nix/store/aaaa...-hello-2.12
|
||||
//! URL: nar/1b2m2y0h...nar.xz
|
||||
//! Compression: xz
|
||||
//! FileHash: sha256:1b2m2y0h...
|
||||
//! FileSize: 54321
|
||||
//! NarHash: sha256:0abcdef...
|
||||
//! NarSize: 123456
|
||||
//! References: bbbb...-glibc-2.37 cccc...-gcc-12.3.0
|
||||
//! Deriver: dddd...-hello-2.12.drv
|
||||
//! Sig: cache.example.com:AAAA...==
|
||||
//! ```
|
||||
//!
|
||||
//! ## Fields
|
||||
//!
|
||||
//! | Field | Required | Description |
|
||||
//! |---------------|----------|-----------------------------------------------------|
|
||||
//! | `StorePath` | yes | Full Nix store path |
|
||||
//! | `URL` | yes | Relative URL to the (possibly compressed) NAR file |
|
||||
//! | `Compression` | no | `xz`, `zstd`, `bzip2`, or `none` (default: `none`) |
|
||||
//! | `FileHash` | yes | Hash of the compressed file on disk |
|
||||
//! | `FileSize` | yes | Size (bytes) of the compressed file |
|
||||
//! | `NarHash` | yes | Hash of the uncompressed NAR archive |
|
||||
//! | `NarSize` | yes | Size (bytes) of the uncompressed NAR archive |
|
||||
//! | `References` | no | Space-separated list of store-path basenames this path depends on |
|
||||
//! | `Deriver` | no | The `.drv` file that produced this output |
|
||||
//! | `Sig` | no | Cryptographic signature(s); may appear multiple times|
|
||||
//!
|
||||
//! This module provides [`NarInfo::parse`] to deserialise the text format and
|
||||
//! a [`fmt::Display`] implementation to serialise it back.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
/// A parsed NARInfo record.
|
||||
///
|
||||
/// Represents all the metadata Nix needs to fetch and verify a single store
|
||||
/// path from a binary cache. Instances are created by parsing the text
|
||||
/// format received from HTTP requests ([`NarInfo::parse`]) and serialised
|
||||
/// back to text via the [`Display`](fmt::Display) implementation when serving
|
||||
/// `GET /{storeHash}.narinfo` responses.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NarInfo {
|
||||
/// Full Nix store path, e.g. `/nix/store/aaaa...-hello-2.12`.
|
||||
pub store_path: String,
|
||||
|
||||
/// Relative URL pointing to the (possibly compressed) NAR archive,
|
||||
/// e.g. `nar/1b2m2y0h...nar.xz`. The client appends this to the
|
||||
/// cache base URL to download the archive.
|
||||
pub url: String,
|
||||
|
||||
/// Compression algorithm applied to the NAR archive on disk.
|
||||
/// Nix uses this to decide how to decompress after downloading.
|
||||
pub compression: Compression,
|
||||
|
||||
/// Content-addressed hash of the compressed file (the file referenced
|
||||
/// by [`url`](Self::url)), usually in the form `sha256:<base32>`.
|
||||
pub file_hash: String,
|
||||
|
||||
/// Size in bytes of the compressed file on disk.
|
||||
pub file_size: u64,
|
||||
|
||||
/// Content-addressed hash of the *uncompressed* NAR archive.
|
||||
/// Nix uses this to verify integrity after decompression.
|
||||
pub nar_hash: String,
|
||||
|
||||
/// Size in bytes of the uncompressed NAR archive.
|
||||
pub nar_size: u64,
|
||||
|
||||
/// Other store paths that this path depends on at runtime, listed as
|
||||
/// basenames (e.g. `bbbb...-glibc-2.37`). An empty `Vec` means the
|
||||
/// path is self-contained.
|
||||
pub references: Vec<String>,
|
||||
|
||||
/// The `.drv` basename that built this output, if known.
|
||||
pub deriver: Option<String>,
|
||||
|
||||
/// Zero or more cryptographic signatures that attest to the
|
||||
/// authenticity of this store path. Each signature is of the form
|
||||
/// `<key-name>:<base64-sig>`. Multiple `Sig` lines are allowed in
|
||||
/// the on-wire format.
|
||||
pub sig: Vec<String>,
|
||||
}
|
||||
|
||||
/// The compression algorithm used for a NAR archive on disk.
|
||||
///
|
||||
/// When a NAR file is stored in the cache it may be compressed to save space
|
||||
/// and bandwidth. The compression type is inferred from the file extension
|
||||
/// (`.xz`, `.zst`, `.bz2`) and recorded in the NARInfo so that clients know
|
||||
/// how to decompress the download.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Compression {
|
||||
/// No compression -- the file is a raw `.nar`.
|
||||
None,
|
||||
/// LZMA2 compression (`.nar.xz`). This is the most common format used
|
||||
/// by `cache.nixos.org`.
|
||||
Xz,
|
||||
/// Zstandard compression (`.nar.zst`). Faster than xz with comparable
|
||||
/// compression ratios; increasingly popular for newer caches.
|
||||
Zstd,
|
||||
/// Bzip2 compression (`.nar.bz2`). A legacy format still seen in some
|
||||
/// older caches.
|
||||
Bzip2,
|
||||
}
|
||||
|
||||
impl NarInfo {
|
||||
/// Parse the line-oriented NARInfo text format into a [`NarInfo`] struct.
|
||||
///
|
||||
/// The format is a series of `Key: Value` lines. Required fields are
|
||||
/// `StorePath`, `URL`, `FileHash`, `FileSize`, `NarHash`, and `NarSize`.
|
||||
/// If any required field is missing, a
|
||||
/// [`CacheError::InvalidNarInfo`](crate::error::CacheError::InvalidNarInfo)
|
||||
/// error is returned.
|
||||
///
|
||||
/// Unknown keys are silently ignored so that forward-compatibility with
|
||||
/// future Nix versions is preserved.
|
||||
pub fn parse(input: &str) -> Result<Self, crate::error::CacheError> {
|
||||
let mut store_path = None;
|
||||
let mut url = None;
|
||||
let mut compression = Compression::None;
|
||||
let mut file_hash = None;
|
||||
let mut file_size = None;
|
||||
let mut nar_hash = None;
|
||||
let mut nar_size = None;
|
||||
let mut references = Vec::new();
|
||||
let mut deriver = None;
|
||||
let mut sig = Vec::new();
|
||||
|
||||
for line in input.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some((key, value)) = line.split_once(": ") {
|
||||
match key {
|
||||
"StorePath" => store_path = Some(value.to_string()),
|
||||
"URL" => url = Some(value.to_string()),
|
||||
"Compression" => {
|
||||
compression = match value {
|
||||
"xz" => Compression::Xz,
|
||||
"zstd" => Compression::Zstd,
|
||||
"bzip2" => Compression::Bzip2,
|
||||
"none" => Compression::None,
|
||||
_ => Compression::None,
|
||||
}
|
||||
}
|
||||
"FileHash" => file_hash = Some(value.to_string()),
|
||||
"FileSize" => file_size = value.parse().ok(),
|
||||
"NarHash" => nar_hash = Some(value.to_string()),
|
||||
"NarSize" => nar_size = value.parse().ok(),
|
||||
"References" => {
|
||||
if !value.is_empty() {
|
||||
references =
|
||||
value.split_whitespace().map(String::from).collect();
|
||||
}
|
||||
}
|
||||
"Deriver" => deriver = Some(value.to_string()),
|
||||
"Sig" => sig.push(value.to_string()),
|
||||
_ => {} // ignore unknown fields for forward-compatibility
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(NarInfo {
|
||||
store_path: store_path.ok_or_else(|| {
|
||||
crate::error::CacheError::InvalidNarInfo("missing StorePath".into())
|
||||
})?,
|
||||
url: url.ok_or_else(|| {
|
||||
crate::error::CacheError::InvalidNarInfo("missing URL".into())
|
||||
})?,
|
||||
compression,
|
||||
file_hash: file_hash.ok_or_else(|| {
|
||||
crate::error::CacheError::InvalidNarInfo("missing FileHash".into())
|
||||
})?,
|
||||
file_size: file_size.ok_or_else(|| {
|
||||
crate::error::CacheError::InvalidNarInfo("missing FileSize".into())
|
||||
})?,
|
||||
nar_hash: nar_hash.ok_or_else(|| {
|
||||
crate::error::CacheError::InvalidNarInfo("missing NarHash".into())
|
||||
})?,
|
||||
nar_size: nar_size.ok_or_else(|| {
|
||||
crate::error::CacheError::InvalidNarInfo("missing NarSize".into())
|
||||
})?,
|
||||
references,
|
||||
deriver,
|
||||
sig,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Serialises the [`NarInfo`] back into the canonical line-oriented text
|
||||
/// format expected by Nix clients.
|
||||
///
|
||||
/// The output is suitable for returning directly as the body of a
|
||||
/// `GET /{storeHash}.narinfo` HTTP response. Optional fields (`References`,
|
||||
/// `Deriver`, `Sig`) are only emitted when present.
|
||||
impl fmt::Display for NarInfo {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
writeln!(f, "StorePath: {}", self.store_path)?;
|
||||
writeln!(f, "URL: {}", self.url)?;
|
||||
writeln!(
|
||||
f,
|
||||
"Compression: {}",
|
||||
match self.compression {
|
||||
Compression::None => "none",
|
||||
Compression::Xz => "xz",
|
||||
Compression::Zstd => "zstd",
|
||||
Compression::Bzip2 => "bzip2",
|
||||
}
|
||||
)?;
|
||||
writeln!(f, "FileHash: {}", self.file_hash)?;
|
||||
writeln!(f, "FileSize: {}", self.file_size)?;
|
||||
writeln!(f, "NarHash: {}", self.nar_hash)?;
|
||||
writeln!(f, "NarSize: {}", self.nar_size)?;
|
||||
if !self.references.is_empty() {
|
||||
writeln!(f, "References: {}", self.references.join(" "))?;
|
||||
}
|
||||
if let Some(ref deriver) = self.deriver {
|
||||
writeln!(f, "Deriver: {}", deriver)?;
|
||||
}
|
||||
for s in &self.sig {
|
||||
writeln!(f, "Sig: {}", s)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
158
crates/jupiter-cache/src/routes.rs
Normal file
158
crates/jupiter-cache/src/routes.rs
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
//! Axum HTTP route handlers for the Nix binary cache protocol.
|
||||
//!
|
||||
//! This module exposes the standard Nix binary cache endpoints so that any
|
||||
//! Nix client (or Hercules CI agent) can interact with Jupiter's built-in
|
||||
//! cache exactly as it would with `cache.nixos.org` or any other
|
||||
//! Nix-compatible binary cache.
|
||||
//!
|
||||
//! ## Endpoints
|
||||
//!
|
||||
//! | Method | Path | Handler | Purpose |
|
||||
//! |--------|--------------------------------|--------------------|--------------------------------------------------|
|
||||
//! | `GET` | `/nix-cache-info` | [`nix_cache_info`] | Return cache metadata (store dir, priority, etc.)|
|
||||
//! | `GET` | `/{storeHash}.narinfo` | [`get_narinfo`] | Fetch NARInfo metadata for a store path hash |
|
||||
//! | `PUT` | `/{storeHash}.narinfo` | [`put_narinfo`] | Upload NARInfo metadata (agent -> cache) |
|
||||
//! | `GET` | `/nar/{filename}` | [`get_nar`] | Download a (possibly compressed) NAR archive |
|
||||
//!
|
||||
//! All handlers receive an `Arc<LocalStore>` via Axum's shared state
|
||||
//! mechanism. Use [`cache_routes`] to obtain a configured [`Router`] that
|
||||
//! can be merged into the main Jupiter server.
|
||||
|
||||
use axum::{
|
||||
extract::{Path, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::get,
|
||||
Router,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::store::LocalStore;
|
||||
|
||||
/// Build an Axum [`Router`] that serves the Nix binary cache protocol.
|
||||
///
|
||||
/// The router expects an `Arc<LocalStore>` as shared state. Callers
|
||||
/// typically do:
|
||||
///
|
||||
/// ```ignore
|
||||
/// let store = Arc::new(LocalStore::new("/var/cache/jupiter", None).await?);
|
||||
/// let app = cache_routes().with_state(store);
|
||||
/// ```
|
||||
///
|
||||
/// The returned router can be nested under a sub-path or merged directly
|
||||
/// into the top-level Jupiter application router.
|
||||
pub fn cache_routes() -> Router<Arc<LocalStore>> {
|
||||
Router::new()
|
||||
.route("/nix-cache-info", get(nix_cache_info))
|
||||
.route(
|
||||
"/{store_hash}.narinfo",
|
||||
get(get_narinfo).put(put_narinfo),
|
||||
)
|
||||
.route("/nar/{filename}", get(get_nar))
|
||||
}
|
||||
|
||||
/// `GET /nix-cache-info` -- return static cache metadata.
|
||||
///
|
||||
/// This is the first endpoint a Nix client hits when it discovers a new
|
||||
/// binary cache. The response is a simple key-value text format:
|
||||
///
|
||||
/// * `StoreDir: /nix/store` -- the Nix store prefix (always `/nix/store`).
|
||||
/// * `WantMassQuery: 1` -- tells the client it is OK to query many
|
||||
/// paths at once (e.g. during `nix-store --query`).
|
||||
/// * `Priority: 30` -- a hint for substitution ordering. Lower
|
||||
/// numbers are preferred. 30 is a reasonable middle-ground that lets
|
||||
/// upstream caches (priority 10-20) take precedence when configured.
|
||||
async fn nix_cache_info() -> impl IntoResponse {
|
||||
(
|
||||
StatusCode::OK,
|
||||
[("Content-Type", "text/x-nix-cache-info")],
|
||||
"StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 30\n",
|
||||
)
|
||||
}
|
||||
|
||||
/// `GET /{store_hash}.narinfo` -- look up NARInfo by store-path hash.
|
||||
///
|
||||
/// `store_hash` is the 32-character base-32 hash that identifies a Nix
|
||||
/// store path (the portion between `/nix/store/` and the first `-`).
|
||||
///
|
||||
/// On success the response has content-type `text/x-nix-narinfo` and
|
||||
/// contains the NARInfo in its canonical text representation. If the hash
|
||||
/// is not present in the cache, a plain 404 is returned so that the Nix
|
||||
/// client can fall through to the next configured substituter.
|
||||
async fn get_narinfo(
|
||||
State(store): State<Arc<LocalStore>>,
|
||||
Path(store_hash): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
match store.get_narinfo(&store_hash).await {
|
||||
Ok(narinfo) => (
|
||||
StatusCode::OK,
|
||||
[("Content-Type", "text/x-nix-narinfo")],
|
||||
narinfo.to_string(),
|
||||
)
|
||||
.into_response(),
|
||||
Err(_) => StatusCode::NOT_FOUND.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `PUT /{store_hash}.narinfo` -- upload NARInfo metadata.
|
||||
///
|
||||
/// Hercules CI agents call this endpoint after building a derivation to
|
||||
/// publish the artefact metadata to the shared cache. The request body
|
||||
/// must be a valid NARInfo text document. The handler parses it to validate
|
||||
/// correctness before persisting it to disk via
|
||||
/// [`LocalStore::put_narinfo`](crate::store::LocalStore::put_narinfo).
|
||||
///
|
||||
/// ## Response codes
|
||||
///
|
||||
/// * `200 OK` -- the NARInfo was stored successfully.
|
||||
/// * `400 Bad Request` -- the body could not be parsed as valid NARInfo.
|
||||
/// * `500 Internal Server Error` -- an I/O error occurred while writing.
|
||||
async fn put_narinfo(
|
||||
State(store): State<Arc<LocalStore>>,
|
||||
Path(store_hash): Path<String>,
|
||||
body: String,
|
||||
) -> impl IntoResponse {
|
||||
match crate::narinfo::NarInfo::parse(&body) {
|
||||
Ok(narinfo) => match store.put_narinfo(&store_hash, &narinfo).await {
|
||||
Ok(()) => StatusCode::OK.into_response(),
|
||||
Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(),
|
||||
},
|
||||
Err(e) => (StatusCode::BAD_REQUEST, e.to_string()).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /nar/{filename}` -- download a NAR archive.
|
||||
///
|
||||
/// `filename` has the form `<narHash>.nar[.xz|.zst]`. The handler strips
|
||||
/// the hash from the filename, delegates to
|
||||
/// [`LocalStore::get_nar`](crate::store::LocalStore::get_nar) to read the
|
||||
/// raw bytes from disk, and returns them with the appropriate `Content-Type`:
|
||||
///
|
||||
/// | Extension | Content-Type |
|
||||
/// |-----------|--------------------------|
|
||||
/// | `.xz` | `application/x-xz` |
|
||||
/// | `.zst` | `application/zstd` |
|
||||
/// | (none) | `application/x-nix-nar` |
|
||||
///
|
||||
/// No server-side decompression is performed; the Nix client handles that
|
||||
/// based on the `Compression` field in the corresponding NARInfo.
|
||||
async fn get_nar(
|
||||
State(store): State<Arc<LocalStore>>,
|
||||
Path(filename): Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
// Extract hash from filename (e.g., "abc123.nar.xz" -> "abc123")
|
||||
let nar_hash = filename.split('.').next().unwrap_or(&filename);
|
||||
match store.get_nar(nar_hash).await {
|
||||
Ok(data) => {
|
||||
let content_type = if filename.ends_with(".xz") {
|
||||
"application/x-xz"
|
||||
} else if filename.ends_with(".zst") {
|
||||
"application/zstd"
|
||||
} else {
|
||||
"application/x-nix-nar"
|
||||
};
|
||||
(StatusCode::OK, [("Content-Type", content_type)], data).into_response()
|
||||
}
|
||||
Err(_) => StatusCode::NOT_FOUND.into_response(),
|
||||
}
|
||||
}
|
||||
178
crates/jupiter-cache/src/store.rs
Normal file
178
crates/jupiter-cache/src/store.rs
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
//! On-disk storage backend for the Nix binary cache.
|
||||
//!
|
||||
//! [`LocalStore`] persists NARInfo files and NAR archives to a local
|
||||
//! directory. The on-disk layout mirrors the URL scheme of the binary cache
|
||||
//! HTTP protocol:
|
||||
//!
|
||||
//! ```text
|
||||
//! <root>/
|
||||
//! aaaa....narinfo # NARInfo for store hash "aaaa..."
|
||||
//! bbbb....narinfo # NARInfo for store hash "bbbb..."
|
||||
//! nar/
|
||||
//! 1b2m2y0h....nar.xz # Compressed NAR archive
|
||||
//! cdef5678....nar.zst # Another archive, zstd-compressed
|
||||
//! abcd1234....nar # Uncompressed NAR archive
|
||||
//! ```
|
||||
//!
|
||||
//! This layout means the cache directory can also be served by any static
|
||||
//! file HTTP server (e.g. nginx) if desired, without any application logic.
|
||||
//!
|
||||
//! ## Concurrency
|
||||
//!
|
||||
//! All public methods are `async` and use `tokio::fs` for non-blocking I/O.
|
||||
//! The struct is designed to be wrapped in an `Arc` and shared across Axum
|
||||
//! handler tasks.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use tokio::fs;
|
||||
|
||||
use crate::error::CacheError;
|
||||
use crate::narinfo::NarInfo;
|
||||
|
||||
/// A filesystem-backed Nix binary cache store.
|
||||
///
|
||||
/// Holds NARInfo metadata files and NAR archives in a directory tree whose
|
||||
/// layout is compatible with the standard Nix binary cache HTTP protocol.
|
||||
/// An optional maximum size (in gigabytes) can be specified at construction
|
||||
/// time to cap disk usage.
|
||||
///
|
||||
/// # Usage within Jupiter
|
||||
///
|
||||
/// When the built-in cache feature is enabled, a `LocalStore` is created at
|
||||
/// server startup, wrapped in an `Arc`, and passed as Axum shared state to
|
||||
/// the route handlers in [`crate::routes`]. Hercules CI agents can then
|
||||
/// `PUT` NARInfo files and NAR archives into the cache and `GET` them back
|
||||
/// when they (or other agents / developer workstations) need to fetch build
|
||||
/// artefacts.
|
||||
pub struct LocalStore {
|
||||
/// Root directory of the cache on disk.
|
||||
path: PathBuf,
|
||||
/// Optional disk-usage cap, in bytes. Derived from the `max_size_gb`
|
||||
/// constructor parameter. Currently stored for future enforcement;
|
||||
/// the quota-checking logic is not yet wired up.
|
||||
#[allow(dead_code)]
|
||||
max_size_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
impl LocalStore {
|
||||
/// Create or open a [`LocalStore`] rooted at `path`.
|
||||
///
|
||||
/// The constructor ensures that both the root directory and the `nar/`
|
||||
/// subdirectory exist (creating them if necessary). The optional
|
||||
/// `max_size_gb` parameter sets an upper bound on total disk usage; pass
|
||||
/// `None` for an unbounded cache.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`CacheError::Io`] if the directories cannot be created.
|
||||
pub async fn new(
|
||||
path: impl Into<PathBuf>,
|
||||
max_size_gb: Option<u64>,
|
||||
) -> Result<Self, CacheError> {
|
||||
let path = path.into();
|
||||
fs::create_dir_all(&path).await?;
|
||||
fs::create_dir_all(path.join("nar")).await?;
|
||||
Ok(Self {
|
||||
path,
|
||||
max_size_bytes: max_size_gb.map(|gb| gb * 1024 * 1024 * 1024),
|
||||
})
|
||||
}
|
||||
|
||||
/// Retrieve and parse the NARInfo for a given store hash.
|
||||
///
|
||||
/// `store_hash` is the first 32 base-32 characters of a Nix store path
|
||||
/// (everything between `/nix/store/` and the first `-`). The method
|
||||
/// reads `<root>/<store_hash>.narinfo` from disk and parses it into a
|
||||
/// [`NarInfo`] struct.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// * [`CacheError::NotFound`] -- the `.narinfo` file does not exist.
|
||||
/// * [`CacheError::InvalidNarInfo`] -- the file exists but cannot be parsed.
|
||||
pub async fn get_narinfo(&self, store_hash: &str) -> Result<NarInfo, CacheError> {
|
||||
let path = self.path.join(format!("{}.narinfo", store_hash));
|
||||
let content = fs::read_to_string(&path)
|
||||
.await
|
||||
.map_err(|_| CacheError::NotFound(store_hash.to_string()))?;
|
||||
NarInfo::parse(&content)
|
||||
}
|
||||
|
||||
/// Write a NARInfo file for the given store hash.
|
||||
///
|
||||
/// Serialises `narinfo` using its [`Display`](std::fmt::Display)
|
||||
/// implementation and writes it to `<root>/<store_hash>.narinfo`. If a
|
||||
/// narinfo for this hash already exists it is silently overwritten.
|
||||
///
|
||||
/// This is the server-side handler for `PUT /{storeHash}.narinfo` --
|
||||
/// Hercules CI agents call this after building a derivation to publish
|
||||
/// the artefact metadata to the shared cache.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`CacheError::Io`] if the file cannot be written.
|
||||
pub async fn put_narinfo(
|
||||
&self,
|
||||
store_hash: &str,
|
||||
narinfo: &NarInfo,
|
||||
) -> Result<(), CacheError> {
|
||||
let path = self.path.join(format!("{}.narinfo", store_hash));
|
||||
fs::write(&path, narinfo.to_string()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read a NAR archive from disk.
|
||||
///
|
||||
/// Because the archive may be stored with any compression extension, this
|
||||
/// method probes for the file with no extension, `.xz`, and `.zst` in
|
||||
/// that order. The first match wins. The raw bytes are returned without
|
||||
/// any decompression -- the HTTP response will carry the appropriate
|
||||
/// `Content-Type` header so the Nix client knows how to handle it.
|
||||
///
|
||||
/// `nar_hash` is the content-address portion of the filename (the part
|
||||
/// before `.nar`).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// * [`CacheError::NotFound`] -- no file matches any of the probed extensions.
|
||||
/// * [`CacheError::Io`] -- the file exists but could not be read.
|
||||
pub async fn get_nar(&self, nar_hash: &str) -> Result<Vec<u8>, CacheError> {
|
||||
// Try multiple extensions to support all compression variants that
|
||||
// may have been uploaded.
|
||||
for ext in &["", ".xz", ".zst"] {
|
||||
let path = self
|
||||
.path
|
||||
.join("nar")
|
||||
.join(format!("{}.nar{}", nar_hash, ext));
|
||||
if path.exists() {
|
||||
return fs::read(&path).await.map_err(CacheError::Io);
|
||||
}
|
||||
}
|
||||
Err(CacheError::NotFound(nar_hash.to_string()))
|
||||
}
|
||||
|
||||
/// Write a NAR archive to the `nar/` subdirectory.
|
||||
///
|
||||
/// `filename` should include the full name with extension, e.g.
|
||||
/// `1b2m2y0h...nar.xz`. The compression variant is implicit in the
|
||||
/// extension; no server-side (de)compression is performed.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`CacheError::Io`] if the file cannot be written.
|
||||
pub async fn put_nar(&self, filename: &str, data: &[u8]) -> Result<(), CacheError> {
|
||||
let path = self.path.join("nar").join(filename);
|
||||
fs::write(&path, data).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check whether a NARInfo file exists for the given store hash.
|
||||
///
|
||||
/// This is a lightweight existence check (no parsing) useful for
|
||||
/// short-circuiting duplicate uploads. Returns `true` if
|
||||
/// `<root>/<store_hash>.narinfo` is present on disk.
|
||||
pub async fn has_narinfo(&self, store_hash: &str) -> bool {
|
||||
self.path
|
||||
.join(format!("{}.narinfo", store_hash))
|
||||
.exists()
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue