Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions src/commands/port.rs
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ pub async fn run(command: PortCommands) -> Result<(), String> {
let egaps = hashfill::extra_gaps(&root);
if !egaps.is_empty() {
println!("hashing {} side files ...", egaps.len());
let mut per_file: std::collections::BTreeMap<PathBuf, Vec<(String, String, String, String)>> =
let mut per_file: std::collections::BTreeMap<PathBuf, Vec<(String, String, Option<String>, String, String)>> =
Default::default();
let mut efailed = 0;
let mut estream = futures::stream::iter(egaps.iter().map(|g| {
Expand All @@ -228,7 +228,7 @@ pub async fn run(command: PortCommands) -> Result<(), String> {
Ok((b3, sha, _)) => per_file
.entry(g.path.clone())
.or_default()
.push((g.url.clone(), g.to.clone(), b3, sha)),
.push((g.url.clone(), g.to.clone(), g.host.clone(), b3, sha)),
Err(e) => {
efailed += 1;
eprintln!(" {} {}: {e}", "FAIL".red(), g.url);
Expand Down Expand Up @@ -260,9 +260,13 @@ pub async fn run(command: PortCommands) -> Result<(), String> {
}
println!("\n{ok}/{} verified against real archive contents", findings.len());
if !unlistable.is_empty() {
// Single-file compression carries no member list, so these
// are unchecked rather than wrong.
println!("\n{} unlistable (single-file compression):", unlistable.len());
// A bare binary, or one compressed on its own, has no members
// to list. The artifact is the file the package installs, so
// there are no interior paths that could be wrong.
println!(
"\n{} not archives, nothing to verify inside:",
unlistable.len()
);
for f in &unlistable {
println!(" {} ({})", f.package, f.host);
}
Expand Down
88 changes: 63 additions & 25 deletions src/port/hashfill.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ use sha2::{Digest, Sha256};

/// A side file declared in pkg.toml that this version has not pinned yet.
pub struct ExtraGap {
/// Which host this gap is for, when the URL is arch-dependent.
pub host: Option<String>,
pub path: std::path::PathBuf,
/// URL the client will fetch it from.
pub url: String,
Expand Down Expand Up @@ -46,27 +48,59 @@ pub fn extra_gaps(root: &Path) -> Vec<ExtraGap> {
paths.sort();
for (v, path) in p.versions.iter().zip(paths) {
for e in &p.pkg.extra {
let pinned = v
.extra
.iter()
.any(|x| x.to == e.to && x.blake3.is_some());
if pinned {
continue;
}
// A vendored licence resolves to this repository rather than
// upstream, which is the point: some hosts rate-limit and some
// upstreams are gone.
let (url, local) = match (&e.url, &e.license) {
(Some(u), _) => (u.replace("${version}", &v.version), None),
(None, Some(spdx)) => (
format!(
"https://raw.githubusercontent.com/pkgforge/soarpkgs/main/licenses/{spdx}.txt"
),
Some(root.join("licenses").join(format!("{spdx}.txt"))),
),
_ => continue,
// A side file whose URL names an architecture is a different
// file on every host, so it is pinned once per host rather
// than once per package.
let hosts: Vec<Option<String>> = match &e.url {
Some(u) if u.contains("${arch}") => p
.pkg
.host
.supported
.iter()
.map(|h| Some(h.clone()))
.collect(),
_ => vec![None],
};
out.push(ExtraGap { path: path.clone(), url, to: e.to.clone(), local });
for host in hosts {
let pinned = v
.extra
.iter()
.any(|x| x.to == e.to && x.host == host && x.blake3.is_some());
if pinned {
continue;
}
// A vendored licence resolves to this repository rather
// than upstream, which is the point: some hosts rate-limit
// and some upstreams are gone.
let (url, local) = match (&e.url, &e.license) {
(Some(u), _) => (u.replace("${version}", &v.version), None),
(None, Some(spdx)) => (
format!(
"https://raw.githubusercontent.com/pkgforge/soarpkgs/main/licenses/{spdx}.txt"
),
Some(root.join("licenses").join(format!("{spdx}.txt"))),
),
_ => continue,
};
// `${arch}` is whatever upstream calls the architecture,
// which is not always what the host is called.
let url = match &host {
Some(h) => {
let raw = h.split('-').next().unwrap_or(h);
let arch =
p.pkg.arch.get(raw).cloned().unwrap_or_else(|| raw.to_string());
url.replace("${arch}", &arch)
}
None => url,
};
out.push(ExtraGap {
host,
path: path.clone(),
url,
to: e.to.clone(),
local,
});
}
}
}
}
Expand Down Expand Up @@ -152,14 +186,18 @@ pub async fn digests(client: &reqwest::Client, url: &str) -> Result<(String, Str
/// Append resolved side files to a version file.
pub fn merge_extras(
path: &Path,
new: &[(String, String, String, String)],
new: &[(String, String, Option<String>, String, String)],
) -> Result<(), String> {
let raw = fs::read_to_string(path).map_err(|e| e.to_string())?;
let mut s = raw.trim_end().to_string();
for (url, to, b3, sha) in new {
s.push_str(&format!(
"\n\n[[extra]]\nurl = {url:?}\nto = {to:?}\nblake3 = {b3:?}\nsha256 = {sha:?}"
));
for (url, to, host, b3, sha) in new {
s.push_str("\n\n[[extra]]");
s.push_str(&format!("\nurl = {url:?}"));
s.push_str(&format!("\nto = {to:?}"));
if let Some(h) = host {
s.push_str(&format!("\nhost = {h:?}"));
}
s.push_str(&format!("\nblake3 = {b3:?}\nsha256 = {sha:?}"));
}
s.push('\n');
fs::write(path, s).map_err(|e| e.to_string())
Expand Down
79 changes: 32 additions & 47 deletions src/port/meta.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ pub struct Entry {
pub homepage: Vec<String>,
pub license: Vec<String>,
pub maintainer: Vec<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub note: Vec<String>,
pub category: Vec<String>,
pub provides: Vec<String>,
Expand Down Expand Up @@ -70,57 +71,36 @@ pub struct Binary {
pub link_as: Option<String>,
}

/// Drop a leading archive-root component.
/// Whether an installed file is a desktop-integration resource rather than an
/// executable.
///
/// Install paths are written against the archive as published, but soar
/// promotes a single top-level directory away before locating binaries. A
/// path with no directory component is already at the root.
fn strip_archive_root(path: &str) -> String {
match path.split_once('/') {
Some((_, rest)) if !rest.is_empty() => rest.to_string(),
_ => path.to_string(),
}
/// soar treats a non-empty `binaries` as the complete list of things to link,
/// so one icon or desktop entry in there stops the actual binary being found.
fn is_resource(name: &str) -> bool {
let ext = name.rsplit_once('.').map(|(_, e)| e.to_ascii_lowercase());
matches!(ext.as_deref(), Some("desktop" | "png" | "svg" | "xpm" | "ico"))
}

/// Expand the two template variables an install path may carry.
fn expand_arch(s: &str, version: &str, arch: &str) -> String {
s.replace("${version}", version).replace("${arch}", arch)
}

/// Rebuild the user-facing note list from the structured fields.
/// Notes a user needs told, and nothing else.
///
/// Notes are presentation, so they are derived here rather than stored once
/// per package in the tree.
fn render_notes(p: &PkgToml, src: &str) -> Vec<String> {
let explicit = &p.pkg.note;
let is_prov = |n: &String| n.starts_with("Official binary from") || n.starts_with("Fetched from");

// A package may carry its own provenance wording; it wins over the
// derived line and keeps the leading position.
let mut out: Vec<String> = explicit.iter().filter(|n| is_prov(n)).cloned().collect();
if out.is_empty() {
out.push(if p.pkg.kind.as_deref() == Some("appimage") {
format!("Fetched from Pre Built Community Created AppImage. Check/Report @ {src}")
} else {
format!("Official binary from {src}")
});
}

if p.pkg.portable {
let suffix = if p.pkg.kind.as_deref() == Some("appimage") {
"Works on AnyLinux"
} else {
"Portable Static Binary"
};
out.push(format!("[PORTABLE] ({suffix})"));
} else {
/// Provenance and portability restate `src_url` and `type`, which the entry
/// already carries, so they are not repeated here as prose. Needing something
/// from the host is the exception: it is a limitation rather than a property,
/// and there is no other field carrying it.
fn render_notes(p: &PkgToml) -> Vec<String> {
let mut out = Vec::new();
if !p.pkg.portable {
out.push(match &p.pkg.portable_reason {
Some(why) => format!("[NOT PORTABLE] {why}"),
None => "[NOT PORTABLE]".to_string(),
});
}

out.extend(explicit.iter().filter(|n| !is_prov(n)).cloned());
out.extend(p.pkg.note.iter().cloned());
out
}

Expand All @@ -136,7 +116,6 @@ pub fn generate(root: &Path, host: &str) -> (Vec<Entry>, Vec<String>) {
}
let fam = p.pkg.family.clone();
let srcs = p.src_urls();
let src0 = srcs.first().cloned().unwrap_or_default();

for v in &pkg.versions {
let Some(url) = v.url.get(host) else { continue };
Expand Down Expand Up @@ -178,20 +157,23 @@ pub fn generate(root: &Path, host: &str) -> (Vec<Entry>, Vec<String>) {
let nested = from.trim_start_matches("*/").contains('/');
(base != *to || nested)
&& !to.eq_ignore_ascii_case("LICENSE")
&& !is_resource(to)
&& *from != "*"
})
.map(|(from, to)| Binary {
// The index is generated per host, so templates
// are expanded here rather than shipped for the
// client to resolve.
// The archive root is promoted away before
// binaries are resolved, so publish the path
// relative to what remains.
source: strip_archive_root(&expand_arch(
from,
&v.version,
&arch_for_host,
)),
// Published as written against the archive. An
// archive with one top-level directory has it
// promoted away before binaries are resolved, so
// the client retries without the leading
// component; stripping it here instead would
// discard the only thing telling two
// architectures apart in a multi-arch archive.
source: expand_arch(from, &v.version, &arch_for_host)
.trim_start_matches("*/")
.to_string(),
// Strip soar's provides markers; link_as is a
// plain filename.
link_as: Some(
Expand All @@ -211,6 +193,9 @@ pub fn generate(root: &Path, host: &str) -> (Vec<Entry>, Vec<String>) {
.extra
.iter()
.filter(|e| e.blake3.is_some() || e.sha256.is_some())
// A side file pinned per host belongs only to that host's
// index; one without a host applies to all of them.
.filter(|e| e.host.as_deref().is_none_or(|h| h == host))
.map(|e| ExtraFile {
url: e.url.clone(),
to: e.to.clone(),
Expand All @@ -221,7 +206,7 @@ pub fn generate(root: &Path, host: &str) -> (Vec<Entry>, Vec<String>) {

{
let prov = provides.clone();
let mut note = render_notes(p, &src0);
let mut note = render_notes(p);
if let Some(n) = &note_src {
note = n.clone();
}
Expand Down
5 changes: 5 additions & 0 deletions src/port/model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,11 @@ pub struct Extra {
pub struct PinnedExtra {
pub url: String,
pub to: String,
/// Set when the file differs per host, as an upstream's per-arch binary
/// does. Absent means it applies to every host, which is the case for a
/// licence.
#[serde(default)]
pub host: Option<String>,
#[serde(default)]
pub blake3: Option<String>,
#[serde(default)]
Expand Down
18 changes: 18 additions & 0 deletions src/port/validate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,24 @@ pub fn run(root: &Path) -> Report {
errors.push(format!("{tag}: hash for {host} with no url"));
}
}

// One URL serving several hosts is only right when the artifact
// holds every architecture and the install map picks between them.
// Without that, one architecture is being handed another's binary.
let distinct: std::collections::BTreeSet<&String> = v.url.values().collect();
if v.url.len() > 1 && distinct.len() == 1 {
let selects_arch = p
.pkg
.source
.as_ref()
.is_some_and(|s| s.install.keys().any(|k| k.contains("${arch}")));
if !selects_arch {
errors.push(format!(
"{tag}: one url for {} hosts and no ${{arch}} in the install map",
v.url.len()
));
}
}
}
}

Expand Down
Loading