diff --git a/.gitignore b/.gitignore index be7ea401..0e11b667 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ src/*/*.lock /*.conf /test /build +*.snap # ides .idea diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index cef57c21..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "[python]": { - "editor.defaultFormatter": "charliermarsh.ruff" - }, - "python.testing.pytestArgs": ["."], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true -} diff --git a/Cargo.lock b/Cargo.lock index a6057d34..c3e64bd6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,7 +56,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -67,7 +67,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -93,12 +93,24 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "2.10.0" @@ -142,6 +154,12 @@ version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cc" version = "1.2.51" @@ -158,6 +176,23 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core", +] + [[package]] name = "chrono" version = "0.4.42" @@ -409,6 +444,7 @@ name = "debmagic" version = "0.0.1-alpha.8" dependencies = [ "anyhow", + "chrono", "clap", "config", "deb822-lossless", @@ -416,14 +452,19 @@ dependencies = [ "debian-control", "debmagic-common", "dirs", + "futures-util", "glob", "ignore", "libc", "md-5", + "regex", + "reqwest", "serde", "serde_json", "sha1", "sha2 0.11.0", + "test-case", + "tokio", "toml", "toml_edit", "uuid", @@ -433,10 +474,16 @@ dependencies = [ name = "debmagic-common" version = "0.0.1-alpha.8" dependencies = [ + "anyhow", "chrono", + "deb822-lossless", + "debian-changelog", + "md-5", "regex", "serde", "serde_json", + "sha1", + "sha2 0.11.0", "test-case", ] @@ -490,7 +537,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -560,6 +607,67 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -577,8 +685,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -589,10 +699,24 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core", + "wasm-bindgen", +] + [[package]] name = "glob" version = "0.3.3" @@ -648,6 +772,45 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "hybrid-array" version = "0.4.15" @@ -657,6 +820,65 @@ dependencies = [ "typenum", ] +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.64" @@ -809,6 +1031,12 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -823,11 +1051,12 @@ checksum = "7ee5b5339afb4c41626dde77b7a611bd4f2c202b897852b4bcf5d03eddc61010" [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ - "once_cell", + "cfg-if", + "futures-util", "wasm-bindgen", ] @@ -867,18 +1096,19 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.178" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libredox" -version = "0.1.11" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df15f6eac291ed1cf25865b1ee60399f57e7c227e7f51bdbd4c5270396a9ed50" +checksum = "6480ccc157a1389bb2e4891b24751b0f798ba640d22386f23143fbcc89da195a" dependencies = [ "bitflags", "libc", + "plain", "redox_syscall", ] @@ -894,6 +1124,12 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + [[package]] name = "md-5" version = "0.11.0" @@ -910,6 +1146,17 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -1021,6 +1268,24 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "potential_utf" version = "0.1.4" @@ -1039,6 +1304,62 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash 2.1.3", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash 2.1.3", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.42" @@ -1054,11 +1375,43 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + [[package]] name = "redox_syscall" -version = "0.6.0" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96166dafa0886eb81fe1c0a388bece180fbef2135f97c1e2cf8302e74b43b5" +checksum = "737970939a87c6fa31e7acad13307bccbb017a073b695b6089a2c484f929e20e" dependencies = [ "bitflags", ] @@ -1103,6 +1456,61 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "ron" version = "0.12.0" @@ -1125,7 +1533,7 @@ checksum = "417a3a9f582e349834051b8a10c8d71ca88da4211e4093528e36b9845f6b5f21" dependencies = [ "countme", "hashbrown 0.14.5", - "rustc-hash", + "rustc-hash 1.1.0", "text-size", ] @@ -1145,12 +1553,59 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "same-file" version = "1.0.6" @@ -1224,6 +1679,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha1" version = "0.11.0" @@ -1269,6 +1736,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.2" @@ -1281,6 +1754,16 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1293,6 +1776,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.111" @@ -1315,6 +1804,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1415,6 +1913,61 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.9.10+spec-1.1.0" @@ -1467,6 +2020,76 @@ version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typeid" version = "1.0.3" @@ -1509,6 +2132,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -1561,6 +2190,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1584,9 +2222,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -1595,11 +2233,21 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1607,26 +2255,68 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.111", + "syn 3.0.3", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "whoami" version = "1.6.1" @@ -1643,7 +2333,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1705,6 +2395,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1714,6 +2413,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winnow" version = "0.7.14" @@ -1790,6 +2553,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.3" diff --git a/Cargo.toml b/Cargo.toml index 14113622..d0abf6c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,9 @@ toml_edit = ">=0.22" uuid = { version = ">=1.10.0", features = ["v4", "v5"] } chrono = { version = ">=0.4.42" } regex = { version = ">=1.12.2" } +reqwest = { version = ">=0.12", default-features = false, features = ["rustls-tls", "stream"] } +futures-util = "0.3" +tokio = { version = ">=1.43", features = ["macros", "rt-multi-thread"] } debian-changelog = { version = ">=0.2.14" } debian-control = { version = ">=0.1.39" } deb822-lossless = { version = ">=0.2" } diff --git a/README.md b/README.md index 13028075..2ecba0ae 100644 --- a/README.md +++ b/README.md @@ -2,54 +2,53 @@ debmagic logo -Unified and future-proof developer tooling for increased productivity in the [Debian](https://debian.org)/[Ubuntu](https://ubuntu.com) ecosystem. +Modern, robust & easy [Debian](https://debian.org)/[Ubuntu](https://ubuntu.com) packaging - while staying backwards compatible. -> [!IMPORTANT] -> Debmagic's goal: make Debian packaging modern, robust & easy - while being backwards compatible. +Debmagic unifies the the packaging experience as a single, streamlined tool. Additionally, `debmagic-pkg` allows you to define package build recipes in Python. -Debmagic has two independent parts: -- tooling to [**build** and **test**](#debmagic-building) packages - - in isolated container environments (lxd, docker, ...) - - `debmagic build`, `debmagic check`, `debmagic test`, `debmagic shell`, ... -- create package [**build instructions**](#debmagic-packaging) - - using Python with `debian/rules.py` (instead of shell & Makefiles) +- **Isolated builds without the setup**: `debmagic build binary` builds a `debian/`-packaged source tree in a temporary container (using LXD, Incus, Docker) +- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; `--shell-on-failure` and `debmagic shell` drop you right where the build broke +- **Test & sign integrated**: `debmagic test` runs the package's autopkgtest tests in a fresh environment, `debmagic sign` GPG-signs `.changes`/`.dsc`/`.buildinfo` on the host +- **Python packaging API**: replace complicated `debian/rules` Makefiles with typed Python `debian/rules.py`, with optional `dh` compatibility -[![GitHub Actions Status](https://github.com/SFTtech/debmagic/actions/workflows/pull_request.yaml/badge.svg)](https://github.com/SFTtech/debmagic/actions/workflows/push_on_main.yaml) +[![CI](https://github.com/SFTtech/debmagic/actions/workflows/pull_request.yaml/badge.svg)](https://github.com/SFTtech/debmagic/actions) +## Quickstart ---- - -## Documentation - -To learn packaging with debmagic, follow **[the documentation!](https://debmagic.readthedocs.io)**. +```shell +cd your-package # any source tree with a debian/ directory +uvx debmagic build binary --driver docker +``` +> [!TIP] +> `debmagic --help` lists everything. -## Debmagic building +### Commands | Command | Goal | | - | - | -| `debmagic build binary` | Build a binary package in a container | -| `debmagic build source` | Create a source package for upload | -| `debmagic test` | Run Debian autopkgtest tests (`debian/tests/`) against a prior build | -| `debmagic check` | Lint the package | - -> [!TIP] -> Want to know more about how to use debmagic to build a package? See [docs/usage/build.md](docs/usage/build.md) for a quickstart. +| `debmagic build binary` | Build a binary package (`.deb`) in an isolated environment | +| `debmagic build source` | Create a source package (`.dsc`) for upload (incl signing) | +| `debmagic test` | Run the package's autopkgtest tests (`debian/tests/`) against a prior build | +| `debmagic shell` | Attach an interactive shell to the build environment | +| `debmagic sign` | GPG-sign a `.changes` file (and its `.dsc`/`.buildinfo`) on the host | +| `debmagic config` | Inspect and edit the effective `debmagic.toml` configuration | +| `debmagic check` | Lint the package *(in progress)* | -## Debmagic packaging +--- -You can use the debmagic API to create package build instructions 🚀 +## Documentation -- build steps in `debian/rules.py`: - To consolidate the build recipes, we use the [Debmagic API modules](packages/debmagic-pkg/src/debmagic/v0/_module/) which provide a unified **high-level interface** to common build tools like `cargo`, `autotools`, `cmake`, `meson`, `go`, `ninja`, `python setup.py` and more. +To learn using debmagic, follow **[the documentation!](https://debmagic.readthedocs.io)**. -Usually, `debian/rules` is written as shell-oneliners in a **Makefile**. +## Debmagic package recipes -Debmagic allows straight-forward conversion to **Python**, which is especially useful if the packaging is more complex, like [openldap](https://git.launchpad.net/ubuntu/+source/openldap/tree/debian/rules?h=ubuntu/resolute-devel), [dovecot](https://git.launchpad.net/ubuntu/+source/dovecot/tree/debian/rules?h=ubuntu/resolute-devel), [samba](https://git.launchpad.net/ubuntu/+source/samba/tree/debian/rules?h=ubuntu/resolute-devel) or [gcc](https://git.launchpad.net/ubuntu/+source/gcc-15/tree/debian/rules?h=ubuntu/resolute-devel). +You can use the debmagic pkg API to create package build instructions (`debian/rules.py`), using [Debmagic API modules](packages/debmagic-pkg/src/debmagic/v0/_module/) for common build tools like `cargo`, `autotools`, `cmake`, `meson`, `go`, `ninja`, `python` `setup.py`/`pyproject.toml` and more. -We provide an optional `dh` sequence backward compatibility [module](packages/debmagic-pkg/src/debmagic/v0/_module/dh.py). +While simple shell oneliners in a state-of-the art `debian/rules` **Makefile** can suffice for simple packages, packaging more complex projects like [openldap](https://git.launchpad.net/ubuntu/+source/openldap/tree/debian/rules?h=ubuntu/resolute-devel), [dovecot](https://git.launchpad.net/ubuntu/+source/dovecot/tree/debian/rules?h=ubuntu/resolute-devel), [samba](https://git.launchpad.net/ubuntu/+source/samba/tree/debian/rules?h=ubuntu/resolute-devel) or [gcc](https://git.launchpad.net/ubuntu/+source/gcc-15/tree/debian/rules?h=ubuntu/resolute-devel) can benefit from a more structured approach with `debmagic`. +To simplify the conversion of existing packages, we provide an optional `dh` sequence backward compatibility [module](packages/debmagic-pkg/src/debmagic/v0/_module/dh.py). ### Example debian/rules.py @@ -87,55 +86,6 @@ def configure(build: Build): pkg.pack() ``` -### debhelper compatibility - -For even more straightforward conversion of `debian/rules` Makefiles, Debmagic can [use `dh`](packages/debmagic-pkg/src/debmagic/v0/_module/dh.py) and provides **dh overrides**: - -```python -from debmagic.v0 import dh - -# specify dh arguments: -dhp = dh.Preset("--with=python3 --builddirectory=build") -pkg = package(preset=dhp) - -# if needed, define optional overrides: -@dhp.override -def dh_auto_install(build: Build): - print("dh override worked :)") - build.cmd("dh_auto_install --max-parallel=1") - -pkg.pack() -``` - -### Custom functions - -To add custom functions directly usable from CLI (like custom `debian/rules` targets for maintainers): - -```python -pkg = package(...) - -@pkg.custom_function -def something_custom(some_param: int, another_param: str = "some default"): - print(f"you passed {some_param=} {another_param=}") - -pkg.pack() -``` - -This function can be directly called with: - -```console -./debian/rules.py something-custom --another-param=test 1337 -``` - -```text -you passed some_param=test another_param=1337 -``` - -And generates automatic help for: - -```console -./debian/rules.py something-custom --help -``` ## Contributing diff --git a/debian/control b/debian/control index 026892aa..bc843afb 100644 --- a/debian/control +++ b/debian/control @@ -26,6 +26,9 @@ Build-Depends: librust-uuid-dev (>=1.10.0), librust-chrono-dev (>=0.4.42), librust-regex-dev (>=1.12.2), + librust-reqwest-dev (>= 0.12), + librust-futures-util-dev (>= 0.3), + librust-tokio-dev (>= 1.43), librust-test-case-dev (>=3.3.1), librust-pyo3-dev (>=0.27.2), librust-deb822-lossless-dev (>=0.5.18), @@ -34,7 +37,7 @@ Build-Depends: librust-ignore-dev (>=0.4.25), librust-md-5-dev (>= 0.11), librust-sha1-dev (>= 0.11), - librust-sha2-dev (>= 0.11) + librust-sha2-dev (>= 0.11), Rules-Requires-Root: no X-Style: black Standards-Version: 4.7.2 @@ -54,11 +57,26 @@ Multi-Arch: foreign Description: Debian build instructions written in Python. Explicit is better than implicit. +Package: debmagic-dpkg-driver +Architecture: all +Depends: + dpkg-dev (>= 1.23.6), + ${misc:Depends} +Recommends: + debmagic-pkg +Description: dpkg build driver for debmagic rules.py packages. + Packages declaring `Build-Driver: debmagic` are built from `debian/rules.py` + instead of a `debian/rules` Makefile. This package provides the + `Dpkg::BuildDriver::Debmagic` perl module that `dpkg-buildpackage` loads to + dispatch the dpkg build targets to the python entry point. + Package: debmagic Architecture: any Depends: ${misc:Depends}, ${shlibs:Depends}, +Suggests: + debmagic-dpkg-driver Description: Debian package building made easy. Holistic cli for the whole debian package building workflow. Built-Using: ${cargo:Built-Using} diff --git a/debian/debmagic-dpkg-driver.install b/debian/debmagic-dpkg-driver.install new file mode 100644 index 00000000..8ccf73e2 --- /dev/null +++ b/debian/debmagic-dpkg-driver.install @@ -0,0 +1 @@ +packages/debmagic-dpkg-driver/Debmagic.pm usr/share/perl5/Dpkg/BuildDriver/ \ No newline at end of file diff --git a/debian/debmagic.1 b/debian/debmagic.1 index f0d3cfee..f1b68553 100644 --- a/debian/debmagic.1 +++ b/debian/debmagic.1 @@ -1,11 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. -.TH DEBMAGIC "1" "July 2026" "debmagic 0.0.1-alpha1" "User Commands" +.TH DEBMAGIC "1" "September 2026" "debmagic 0.0.1-alpha.8" "User Commands" .SH NAME -debmagic \- build Debian packages in isolated environments +debmagic \- manual page for debmagic 0.0.1-alpha.8 .SH SYNOPSIS .B debmagic [\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION +tooling for building and packaging Debian/Ubuntu packages in isolated environments .SS "Commands:" .TP build @@ -15,23 +16,49 @@ shell Open an interactive shell to the currently active build environment .TP test -Run tests +Run the package's declared Debian autopkgtest tests against a prior build .TP check Check the project .TP +sign +GPG\-sign a .changes file (and its .dsc/.buildinfo) on the host +.TP +upload +Upload a .changes file (and everything it references) to an upload target, dput\-style +.TP +config +Inspect the debmagic configuration +.TP +upstream +Query and switch upstream versions +.TP version Show version information .TP help Print this message or the help of the given subcommand(s) .SH OPTIONS -.TP +.HP \fB\-c\fR, \fB\-\-config\fR +.IP Path to config file -.TP +.HP +\fB\-\-color\fR +.IP +When to colorize output: 'auto' (default) colors on a terminal and respects NO_COLOR, 'always' forces color, 'never' disables it +.IP +Possible values: +\- auto: Color when stderr is a terminal and `NO_COLOR` is unset +\- always: Always color, even when piped or `NO_COLOR` is set +\- never: Never color +.IP +[default: auto] +.HP \fB\-h\fR, \fB\-\-help\fR -Print help -.TP +.IP +Print help (see a summary with '\-h') +.HP \fB\-V\fR, \fB\-\-version\fR +.IP Print version diff --git a/debian/rules b/debian/rules index 2d216ecc..303d7289 100755 --- a/debian/rules +++ b/debian/rules @@ -1,71 +1,12 @@ -#!/usr/bin/env python3 -import shutil -import os +#!/usr/bin/make -f -import sys -from pathlib import Path +# This is a self-build shim: All dpkg build targets are forwarded to the +# real packaging logic in debian/rules.py. +# d/control `Build-driver: debmagic`` (debmagic-dpkg-driver provides Dpkg::BuildDriver::Debmagic) can directly invoke debian/rules.py once installed. -packages_root = Path(__file__).parent.parent / "packages" -sys.path.append(str(packages_root / "debmagic-pkg" / "src")) +PYTHON ?= python3 -from debmagic.v0 import package, dh, Build +%: + $(PYTHON) debian/rules.py $@ -dhp = dh.Preset(dh_args=["--with", "python3", "--buildsystem=pybuild"]) - -pkg = package( - preset=[dhp], -) - -# TODO: add this section to our rust debmagic module to make setting this up less annoying -os.environ.update({ - "PATH": f"/usr/share/cargo/bin:{os.environ['PATH']}", - "CARGO": "/usr/share/cargo/bin/cargo", - "CARGO_HOME": f"{pkg.base_dir}/debian/cargo_home", - "CARGO_REGISTRY": f"{pkg.base_dir}/debian/cargo_registry", - "DEB_CARGO_CRATE": f"{pkg.build_env.DEB_SOURCE}_{pkg.build_env.DEB_VERSION_UPSTREAM}" -}) - -packages = { - "debmagic": ("packages/debmagic", "debmagic"), - "debmagic-pkg": ("packages/debmagic-pkg", "debmagic-pkg"), -} - -cargo_lock = Path("Cargo.lock") -cargo_lock_saved = Path("Cargo.lock.saved") - -def dh_auto(build: Build, stage: str, use_destdir: bool = False): - for pkg_name, (path, python_pkg_name) in packages.items(): - destdir = f" --destdir debian/{pkg_name} " if use_destdir else "" - build.cmd(f"{stage} -p {pkg_name} --sourcedirectory {path} --buildsystem=pybuild {destdir} -- --name {python_pkg_name}") - -@dhp.override -def dh_auto_configure(build: Build): - build.cmd("cargo prepare-debian debian/cargo_registry --link-from-system", cwd="packages/debmagic") - dh_auto(build, "dh_auto_configure") - - -@dhp.override -def dh_auto_build(build: Build): - if cargo_lock.is_file(): - shutil.move(cargo_lock, cargo_lock_saved) - dh_auto(build, "dh_auto_build") - - -@dhp.override -def dh_auto_install(build: Build): - dh_auto(build, "dh_auto_install", use_destdir=True) - - -@dhp.override -def dh_auto_test(build: Build): - dh_auto(build, "dh_auto_test") - - -@dhp.override -def dh_auto_clean(build: Build): - if cargo_lock_saved.is_file(): - shutil.move(cargo_lock_saved, cargo_lock) - dh_auto(build, "dh_auto_clean") - - -pkg.pack() +.PHONY: clean build build-arch build-indep binary binary-arch binary-indep diff --git a/debian/rules.py b/debian/rules.py new file mode 100755 index 00000000..9c3bb05d --- /dev/null +++ b/debian/rules.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +import shutil +import os + +import sys +from pathlib import Path + +packages_root = Path(__file__).parent.parent / "packages" +sys.path.append(str(packages_root / "debmagic-pkg" / "src")) + +from debmagic.v0 import package, dh, Build + +dhp = dh.Preset(dh_args=["--with", "python3", "--buildsystem=pybuild"]) + +pkg = package( + preset=[dhp], +) + +# TODO: add this section to our rust debmagic module to make setting this up less annoying +os.environ.update({ + "PATH": f"/usr/share/cargo/bin:{os.environ['PATH']}", + "CARGO": "/usr/share/cargo/bin/cargo", + "CARGO_HOME": f"{pkg.base_dir}/debian/cargo_home", + "CARGO_REGISTRY": f"{pkg.base_dir}/debian/cargo_registry", + "DEB_CARGO_CRATE": f"{pkg.build_env.DEB_SOURCE}_{pkg.build_env.DEB_VERSION_UPSTREAM}" +}) + +packages = { + "debmagic": ("packages/debmagic", "debmagic"), + "debmagic-pkg": ("packages/debmagic-pkg", "debmagic-pkg"), +} + +cargo_lock = Path("Cargo.lock") +cargo_lock_saved = Path("Cargo.lock.saved") + +def dh_auto(build: Build, stage: str, use_destdir: bool = False): + for pkg_name, (path, python_pkg_name) in packages.items(): + destdir = f" --destdir debian/{pkg_name} " if use_destdir else "" + build.cmd(f"{stage} -p {pkg_name} --sourcedirectory {path} --buildsystem=pybuild {destdir} -- --name {python_pkg_name}") + +@dhp.override +def dh_auto_configure(build: Build): + build.cmd("cargo prepare-debian debian/cargo_registry --link-from-system", cwd="packages/debmagic") + dh_auto(build, "dh_auto_configure") + + +@dhp.override +def dh_auto_build(build: Build): + if cargo_lock.is_file(): + shutil.move(cargo_lock, cargo_lock_saved) + dh_auto(build, "dh_auto_build") + + +@dhp.override +def dh_auto_install(build: Build): + dh_auto(build, "dh_auto_install", use_destdir=True) + + +@dhp.override +def dh_auto_test(build: Build): + dh_auto(build, "dh_auto_test") + + +@dhp.override +def dh_auto_clean(build: Build): + if cargo_lock_saved.is_file(): + shutil.move(cargo_lock_saved, cargo_lock) + dh_auto(build, "dh_auto_clean") + + +pkg.pack() \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index ffd3ac8d..ceb0e54a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,6 +23,8 @@ "html_image", "attrs_inline", ] +# generate heading ids for intra-page anchor links +myst_heading_anchors = 3 templates_path = ["_templates"] exclude_patterns = ["_build"] diff --git a/docs/develop/index.md b/docs/develop/index.md index 3bc9d422..523f3809 100644 --- a/docs/develop/index.md +++ b/docs/develop/index.md @@ -8,6 +8,7 @@ codebase, and some guidelines for how you can contribute. Prerequisites: - Debian >= trixie, either roll your own environment or to get started faster use the [devcontainer](https://containers.dev/) +- Rust >= edition 2024 - Python >= 3.12 - [UV](https://docs.astral.sh/uv/) @@ -18,6 +19,26 @@ uv sync uv run pre-commit install ``` +Build and test: + +```shell +# Rust CLI and shared crate +cargo build +cargo test +cargo clippy --workspace --all-targets --all-features --locked -- -D warnings + +# Python packaging API +uv run pytest --ignore tests/integration . +uv run ruff check +uv run ty check . +``` + +Integration tests (end-to-end builds of real packages): + +```shell +uv run pytest tests/integration +``` + Build the documentation ```shell @@ -27,6 +48,18 @@ uv run sphinx-build docs docs/_build uv run sphinx-autobuild docs docs/_build ``` +## Architecture: `debmagic-common` vs `debmagic` + +We separate between the Debian package model from packaging actions: + +- `packages/debmagic-common` — **pure Debian packaging models, their parsing, and their in-place transformation**, with no I/O side effects. +The `package::Package` is the entry point; `debian::*` holds all substructure models (version, changelog, source format, ...). +- `packages/debmagic` — **packaging CLI** that does all IO (and manages environments) while using the models from `debmagic-common`. + +The layering rule: **common owns bytes ↔ structure (parse, transform, serialize); consumers own paths ↔ bytes**. +Common never touches a file descriptor — a consumer that wants mmap hands common the `&str`/`&[u8]` and writes the +serialized result itself. + ```{toctree} ``` diff --git a/docs/index.md b/docs/index.md index 6e2e4f15..bff6dd8e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,6 +2,34 @@ ![debmagic image](../assets/debmagic-logo.svg){width=250px align=center} +Modern, robust & easy [Debian](https://debian.org)/[Ubuntu](https://ubuntu.com) packaging - while staying backwards compatible. + +Debmagic unifies the the packaging experience as a single, streamlined tool. Additionally, `debmagic-pkg` allows you to define package build recipes in Python. +Working with Debian packages without `debmagic` means juggling with several independent tools like `sbuild`/`pbuilder` chroots, `dch`, `dpkg-buildpackage`, `debsign`, `autopkgtest` virt-backends, `dput`, `lintian`, and various custom glue scripts. +Debmagic unifies this into one CLI with useful defaults: + +- **Isolated builds without the setup**: `debmagic build binary` builds a `debian/`-packaged source tree in a temporary container (using LXD, Incus, Docker) +- **Fast iteration**: `--persistent`/`--incremental` reuse the environment and sync only source changes; `--shell-on-failure` and `debmagic shell` drop you right where the build broke +- **Test & sign integrated**: `debmagic test` runs the package's autopkgtest tests in a fresh environment, `debmagic sign` GPG-signs `.changes`/`.dsc`/`.buildinfo` on the host +- **Package uploading**: `debmagic upload` (to `debian`, `ubuntu`, `ppa:user/repo`) pushes signed builds to archives and PPAs via scp/sftp, with configurable pre-upload checks +- **Upstream tracking**: `debmagic upstream list`/`switch` replace uscan/uupdate, reading the package's existing `debian/watch` and `debian/copyright` +- **Python packaging API**: replace complicated `debian/rules` Makefiles with typed Python `debian/rules.py`, with optional `dh` compatibility + +[![CI](https://github.com/SFTtech/debmagic/actions/workflows/pull_request.yaml/badge.svg)](https://github.com/SFTtech/debmagic/actions) + +## Quickstart + +```shell +cd your-package # any source tree with a debian/ directory +uvx debmagic build binary --driver docker +``` + +> [!TIP] +> `debmagic --help` lists everything. + + +Then dive into the pages below — start with [Getting started](usage/getting-started.md). + ```{toctree} :hidden: :caption: Usage @@ -10,7 +38,10 @@ usage/getting-started.md usage/build.md usage/test.md usage/source.md +usage/upload.md +usage/upstream.md usage/config.md +usage/packaging.md usage/modules/index.md ``` diff --git a/docs/usage/build.md b/docs/usage/build.md index bc0f7767..d5e5762a 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -29,7 +29,9 @@ debmagic build binary --driver lxd \ | `--sign` | [GPG-sign the resulting `.changes`/`.dsc`/`.buildinfo`](#signing) | | `--clean` | Run [`debian/rules clean` before building](#cleaning) | | `--debug-symbols` | [Build the automatic `-dbgsym` debug symbol packages](#building-debug-symbol-packages) | +| `--test` | [Run the package's test suite](#running-tests) | | `--apt-mirror ` | [Mirror URL](#mirror-selection) | +| `--apt-update-age ` | [When a persistent environment re-runs `apt-get update`](#apt-update-age) | | `--source-dir ` | Directory containing the `debian/` package directory | | `--output-dir ` | Directory to put the resulting build artifacts | | `--shell-on-failure` | On build failure, drop into an interactive shell in the build environment when stdout is a TTY | @@ -75,6 +77,20 @@ debmagic build binary --driver lxd --apt-mirror http:///ubuntu ... You can persistently set this flag in `.config/debmagic/config.toml`. +## Apt update age + +Fresh environments always run `apt-get update` once on creation. +When a persistent environment is reused, `--apt-update-age` decides whether the apt index is refreshed again: + +| Value | Behavior | +|---|---| +| `now` | Update before every build | +| `never` | Only the initial update on creation | +| `1d` (default), `12h`, `30m`, … | Update again once the last one is older than this | + +The default `1d` matches a typical developer machine's daily apt refresh: repeated builds stay fast, while the index can't go arbitrarily stale. +Persist the setting as `apt_update_age = "1d"` in [`debmagic.toml`](config.md). + ## Source file staging Before building, `debmagic` stages the source tree into the build environment. @@ -118,10 +134,13 @@ The preserved build tree is kept even when the environment itself is *not* reuse Only needed when `debian/changelog`'s top entry doesn't unambiguously determine the target: pass `--distro ` (e.g. `--distro noble`, `--distro trixie`). If the changelog has a single unambiguous entry, omit it. +`--distro` also overrides the changelog's distribution: you can use it to rebuild a package released for an older release on a newer one, or to attempt a backport. +On the Bare driver the target must still match the host's os-release; pass `--bare-ignore-release` to build for a different suite anyway, with the host providing the build dependencies itself. + Suite aliases in the changelog (or via `--distro`) resolve to a concrete release: Debian `stable` / `oldstable` / `sid` (→ `unstable`), and Ubuntu `devel`. Alias targets are updated manually when Debian/Ubuntu roll. -Non-Debian/Ubuntu suites (still apt/dpkg-based) are supported when declared for the active container Driver via `base_images`, e.g. `driver.docker.base_images = { "yocto:kirkstone" = "my-registry/yocto-kirkstone:latest" }`. The changelog/`--distro` value stays the bare codename (`kirkstone`). On the Bare driver, the host `/etc/os-release` must match: built-in Debian/Ubuntu need matching `ID` and codename; other suites need a matching `VERSION_CODENAME` only. +Non-Debian/Ubuntu suites (still apt/dpkg-based) are supported when declared for the active container Driver via `base_images`, e.g. `driver.docker.base_images = { "yocto:kirkstone" = "my-registry/yocto-kirkstone:latest" }`. The changelog/`--distro` value stays the bare codename (`kirkstone`). On the Bare driver, binary builds require the host `/etc/os-release` to match: built-in Debian/Ubuntu need matching `ID` and codename, other suites a matching `VERSION_CODENAME`. ## Proposed dependencies @@ -139,6 +158,17 @@ debmagic build binary --debug-symbols --output-dir /tmp/out Or set `build_debug_symbols = true` in the [`debmagic.toml`](config.md). +## Running tests + +By default the build runs the package's test suite (the `test` stage of `debian/rules.py`, or `dh_auto_test` via the dh preset). +Pass `--test=false` to skip it for one invocation, or set `run_test = false` in the [`debmagic.toml`](config.md): + +```shell +debmagic build binary --test=false +``` + +This exports `DEB_BUILD_OPTIONS=nocheck`, the standard dpkg mechanism: dpkg-buildpackage propagates it into the build, debmagic's `test` stage is skipped, and classic debhelper packages skip `dh_auto_test` as usual. + ## Signing `--sign` GPG-signs the resulting `.changes`/`.dsc`/`.buildinfo` after building — mainly useful for [source builds destined for Launchpad](source.md#uploading-to-launchpad), but works for binary builds too. @@ -150,7 +180,7 @@ Children are signed first (`.dsc`, then `.buildinfo`) and the `.changes` checksu | `--sign` | `sign.source` | Sign after building; `--sign=false` skips it for one invocation | | `--sign-key ` | `sign.key` | Key ID/fingerprint/email; defaults to the `Changed-By:`/`Maintainer:` address of the file being signed | | `--sign-tool ` | `sign.tool` | OpenPGP implementation: `gpg` (default), `sequoia` (sq), or `custom` | -| `--sign-command ` | `sign.command` | Custom signing command for `--sign-tool custom` (see below) | +| `--sign-command ` | `sign.sign_command` | Custom signing command for `--sign-tool custom` (see below) | | `--sign-notify` | `sign.notify` | Desktop notification + terminal bell just before signing, so a hardware-key touch prompt isn't missed after a long build | A custom signing command runs without a shell and must write the clearsigned result to stdout. diff --git a/docs/usage/config.md b/docs/usage/config.md index c4c0c68d..c2c20ce0 100644 --- a/docs/usage/config.md +++ b/docs/usage/config.md @@ -24,6 +24,7 @@ All keys are optional. | `driver.persistent` | bool | `false` | `--persistent` | Keep and reuse the build environment across runs instead of tearing it down. | | `driver.apt_mirror` | string | — | `--apt-mirror` | Mirror used for build-dependency resolution. Not used by the `bare` driver. | | `driver.proposed` | bool | `false` | `--proposed` | Also enable the `-proposed` pocket. Not used by the `bare` driver. | +| `driver.apt_update_age` | string | `"1d"` | `--apt-update-age` | When a persistent environment re-runs `apt-get update`: `"now"` (every build), `"never"` (only on creation), or a maximum index age like `"1d"`, `"12h"`, `"30m"`. Fresh environments always update once. Not used by the `bare` driver. | | `driver.docker.base_images` | map | — | — | Base image per distro, keyed by `":"` (e.g. `"debian:trixie"`). Falls back to `docker.io/:`. For non-Debian/Ubuntu suites (e.g. `"yocto:kirkstone"`), the map entry is what makes the suite a known DistroVersion for Docker builds. | | `driver.lxd.project` | string | — | — | LXD/Incus project to use. | | `driver.lxd.base_images` | map | — | — | Base image per distro, keyed by `":"`. Falls back to the driver's default remote image. Same custom-suite registry role as Docker's map for LXD/Incus. | @@ -32,14 +33,17 @@ All keys are optional. | `incremental` | bool | `false` | `--incremental` | Retain the environment and sync only source changes, preserving generated files. Binary-only; implies `persistent`; incompatible with `clean`. | | `source_sync_mode` | enum | `tracked` | `--source-sync` | Which source files are staged (see below). | | `build_debug_symbols` | bool | `false` | `--debug-symbols` | Build the automatic `-dbgsym` debug symbol package. | +| `run_test` | bool | `true` | `--test` | Run the package's test suite during the build; `false` exports `DEB_BUILD_OPTIONS=nocheck` so tests are skipped. | | `sign.source` | bool | `false` | `--sign` | Sign the resulting `.changes`/`.dsc` (see below). | | `sign.key` | string | — | `--sign-key` | GPG key ID/email to sign with; falls back to the Changed-By/Maintainer address. | -| `sign.tool` | enum | `gpg` | `--sign-tool` | OpenPGP implementation: `gpg`, `sequoia` (sq) or `custom` (uses `sign.command`). | -| `sign.command` | string | — | `--sign-command` | Custom signing command for `sign.tool = "custom"`, run without a shell with `{file}`/`{key}`/`{email}` placeholders; writes the clearsigned result to stdout. | +| `sign.tool` | enum | `gpg` | `--sign-tool` | OpenPGP implementation: `gpg`, `sequoia` (sq) or `custom` (uses `sign.sign_command`). | +| `sign.sign_command` | string | — | `--sign-command` | Custom signing command for `sign.tool = "custom"`, run without a shell with `{file}`/`{key}`/`{email}` placeholders; writes the clearsigned result to stdout. | | `sign.notify` | bool | `false` | `--sign-notify` | Send a desktop notification via `notify-send` just before signing, so a hardware-key touch prompt isn't missed. | | `clean` | bool | `false` | `--clean` | Run `debian/rules clean` before building. Disabled by default; incompatible with `incremental`. | | `shell_on_failure` | bool | `false` | `--shell-on-failure` | On build or test failure, drop into an interactive shell in the environment when stdout is a TTY. | | `host_arch_variant` | string | — | `--host-arch-variant` | Build for a dpkg architecture variant (e.g. `"amd64v3"` on Ubuntu) -> `DEB_HOST_ARCH_VARIANT`. | +| `upstream.verify_signatures` | bool | `true` | `--no-signature-check` | Verify the upstream tarball's PGP signature against `debian/upstream/signing-key.asc` during `upstream switch` (see [Upstream](upstream.md#signature-verification)). | +| `upload.targets` | map | — | `debmagic upload ` | Named upload targets for [uploading](upload.md), merged field-by-field over the builtins (`ppa`, `ubuntu`, `debian`). | ### `source_sync_mode` @@ -55,12 +59,37 @@ All keys are optional. |---|---|---|---|---| | `source` | bool | `false` | `--sign` | Sign the source package (`.changes`/`.dsc`) after building. | | `key` | string | — | `--sign-key` | GPG key ID/email to sign with; falls back to the Changed-By/Maintainer address. | -| `tool` | enum | `gpg` | `--sign-tool` | OpenPGP implementation: `gpg`, `sequoia` (sq) or `custom` (uses `command`). | -| `command` | string | — | `--sign-command` | Custom signing command for `tool = "custom"`, like `debsign`'s `-p`. | +| `tool` | enum | `gpg` | `--sign-tool` | OpenPGP implementation: `gpg`, `sequoia` (sq) or `custom` (uses `sign_command`). | +| `sign_command` | string | — | `--sign-command` | Custom signing command for `tool = "custom"`, like `debsign`'s `-p`. | | `notify` | bool | `false` | `--sign-notify` | Desktop notification via `notify-send` before signing. | - +| `verify_command` | string | — | `--verify-command` (upstream switch) | Custom verification command for `sign.tool = "custom"`, run without a shell with `{file}`/`{signature}`/`{keyring}` placeholders; without `{signature}` the signature path is appended. Falls back to `sign_command`. | Signing always runs on the host with your gpg keyring — see [Signing](build.md#signing). +### `orig_tarball` + +Where `debmagic build source` fetches the `orig` tarball from when it isn't already in the output dir or next to the source tree — see [Upstream](upstream.md#orig-tarballs-in-builds). + +| Key | Type | Default | Description | +|---|---|---|---| +| `method` | enum | `launchpad` | `launchpad` (the distro archives via Launchpad's download URLs: Ubuntu first, then Debian), `debian` (Debian's own archive pool, without Launchpad), `ubuntu` (Ubuntu's own archive pool, without Launchpad), `custom` (requires `command`) or `disabled` (never fetch; build only with tarballs found locally). | +| `command` | string | — | Custom fetch command for `method = "custom"`, run via `sh -c` in the source dir with `{name}`/`{version}`/`{upstream_version}`/`{source_dir}`/`{output_dir}` placeholders. | +| `debian_mirror` | string | `https://deb.debian.org/debian` | Mirror root for `method = "debian"`, like an apt sources entry; `/pool` is appended. | +| `ubuntu_mirror` | string | `http://archive.ubuntu.com/ubuntu` | Mirror root for `method = "ubuntu"`; `/pool` is appended. | + +### `upload` + +See [Uploading](upload.md) for target resolution, builtins and pre-upload checks. + +| Key | Type | Default | Description | +|---|---|---|---| +| `targets..method` | enum | `scp` | Upload method: `scp` or `sftp`. | +| `targets..server` | string | — | Host to upload to. | +| `targets..incoming` | string | — | Remote directory; supports the `{target}` placeholder. | +| `targets..login` | string | ssh config | Login on the remote host; falls back to the ssh config user, then the local user. The Launchpad/Debian upload hosts expect your own username with a registered SSH key. | +| `targets..port` | int | — | Remote port. | +| `targets..tofu_hostkey` | bool | `true` | Trust the host key on first use (ssh's `StrictHostKeyChecking=accept-new`). The archive upload hosts publish stable keys; disable to require a known_hosts entry. | +| `targets..pre_upload_commands` | list | `[]` | Commands run via `sh -c` before uploading; non-zero aborts the upload. | + ## Example diff --git a/docs/usage/getting-started.md b/docs/usage/getting-started.md index 30415176..8052f56e 100644 --- a/docs/usage/getting-started.md +++ b/docs/usage/getting-started.md @@ -10,16 +10,18 @@ cargo install debmagic ### Pip +Run it directly with [`uv`](https://docs.astral.sh/uv/): + ```shell -pip install debmagic +uvx debmagic ``` -or run it directly with uv: - +or install using `pip`: ```shell -uvx debmagic +pip install debmagic ``` + ### Debian / Ubuntu - Soon (tm) ```shell @@ -30,90 +32,20 @@ apt install debmagic `debmagic build` builds *any* Debian-packaged source tree inside a throwaway build environment, driven by a build driver. -To learn about `debmagic build`, see the [Building packages](build.md) page. - -## Example debian/rules.py - -Python `debian/rules.py` equivalent of [Ubuntu 24.04 htop](https://git.launchpad.net/ubuntu/+source/htop/tree/debian/rules?h=ubuntu/noble): - -```python -#!/usr/bin/env python3 - -from debmagic.v0 import Build, autotools, dh, package - -pkg = package( - preset=[dh], - maint_options="hardening=+all", -) - -if pkg.buildflags.DEB_HOST_ARCH_OS == "linux": - configure_params = ["--enable-affinity", "--enable-delayacct"] -else: - configure_params = ["--enable-hwloc"] - -# hurd-i386 can open /proc (nothing there) and /proc/ which works -if pkg.buildflags.DEB_HOST_ARCH_OS == "hurd": - configure_params += ["--with-proc=/proc/"] -else: - configure_params += ["--enable-sensors"] - - -@pkg.stage -def configure(build: Build): - autotools.configure( - build, - ["--enable-openvz", "--enable-vserver", "--enable-unicode", *configure_params], - ) - -pkg.pack() -``` - -### debhelper compatibility - -Debmagic can use `dh` and provides **dh overrides** as common in `debian/rules` Makefiles: - -```python -from debmagic.v0 import dh - -# specify dh arguments: -dhp = dh.Preset("--with=python3 --builddirectory=build") -pkg = package(preset=dhp) - -# define optional overrides: -@dhp.override -def dh_auto_install(build: Build): - print("dh override worked :)") - build.cmd("dh_auto_install --max-parallel=1") - -pkg.pack() -``` - -### Custom functions - -To add custom functions directly usable from CLI (like custom `debian/rules` targets for maintainers): - -```python -pkg = package(...) - -@pkg.custom_function -def something_custom(some_param: int, another_param: str = "some default"): - print(f"you passed {some_param=} {another_param=}") - -pkg.pack() -``` - -This function can be directly called with: - -```console -./debian/rules.py something-custom --another-param=test 1337 +```shell +cd your-package # any source tree with a debian/ directory +debmagic build binary --driver docker ``` -```text -you passed some_param=test another_param=1337 -``` +The driver picks the isolation technology — `lxd`, `incus`, `docker` (full container isolation) or `bare` (no isolation, for disposable/CI environments). +There's no auto-detection; pass one explicitly or set it in a [`debmagic.toml`](config.md). -And generates automatic help for: +From here: -```console -./debian/rules.py something-custom --help -``` +- [Building packages](build.md) — all `debmagic build` options: drivers, distro selection, incremental builds, signing, ... +- [Running package tests](test.md) — `debmagic test` against a prior build +- [Building source packages](source.md) — `debmagic build source` and uploading to Launchpad +- [Uploading](upload.md) — `debmagic upload`, upload targets and pre-upload checks +- [Upstream management](upstream.md) — `debmagic upstream list`/`switch` for version bumps & backports +- [Configuration](config.md) — persistent settings in `debmagic.toml` +- [Creating package recipes](packaging.md) — writing `debian/rules.py` equivalents with `debmagic-pkg` \ No newline at end of file diff --git a/docs/usage/packaging.md b/docs/usage/packaging.md index d056dd0e..05d919d3 100644 --- a/docs/usage/packaging.md +++ b/docs/usage/packaging.md @@ -2,4 +2,113 @@ `debmagic-pkg` allows you to write a package build recipe in Python. -The documentation is not yet existent. \ No newline at end of file + +## Example debian/rules.py + +Python `debian/rules.py` equivalent of [Ubuntu 24.04 htop](https://git.launchpad.net/ubuntu/+source/htop/tree/debian/rules?h=ubuntu/noble): + +```python +#!/usr/bin/env python3 + +from debmagic.v0 import Build, autotools, dh, package + +pkg = package( + preset=[dh], + maint_options="hardening=+all", +) + +if pkg.buildflags.DEB_HOST_ARCH_OS == "linux": + configure_params = ["--enable-affinity", "--enable-delayacct"] +else: + configure_params = ["--enable-hwloc"] + +# hurd-i386 can open /proc (nothing there) and /proc/ which works +if pkg.buildflags.DEB_HOST_ARCH_OS == "hurd": + configure_params += ["--with-proc=/proc/"] +else: + configure_params += ["--enable-sensors"] + + +@pkg.stage +def configure(build: Build): + autotools.configure( + build, + ["--enable-openvz", "--enable-vserver", "--enable-unicode", *configure_params], + ) + +pkg.pack() +``` + + +### debhelper compatibility + +For even more straightforward conversion of `debian/rules` Makefiles, Debmagic can run [`dh` sequences](packages/debmagic-pkg/src/debmagic/v0/_module/dh.py) and provides **dh overrides**: + +```python +from debmagic.v0 import dh + +# specify dh arguments: +dhp = dh.Preset("--with=python3 --builddirectory=build") +pkg = package(preset=dhp) + +# if needed, define optional overrides: +@dhp.override +def dh_auto_install(build: Build): + print("dh override worked :)") + build.cmd("dh_auto_install --max-parallel=1") + +pkg.pack() +``` + +### Custom functions + +To add custom functions directly usable from CLI (like custom `debian/rules` targets for maintainers): + +```python +pkg = package(...) + +@pkg.custom_function +def something_custom(some_param: int, another_param: str = "some default"): + print(f"you passed {some_param=} {another_param=}") + +pkg.pack() +``` + +This function can be directly called with: + +```console +./debian/rules.py something-custom --another-param=test 1337 +``` + +```text +you passed some_param=test another_param=1337 +``` + +And generates automatic help for: + +```console +./debian/rules.py something-custom --help +``` + + +## Building with dpkg-buildpackage + +A `rules.py` package builds with plain `dpkg-buildpackage` if it also ships a `debian/rules` Makefile shim forwarding all targets to `rules.py`: + +```make +#!/usr/bin/make -f +%: + python3 debian/rules.py $@ +``` + +Alternatively, build-depend on `debmagic-dpkg-driver` and let dpkg invoke `rules.py` directly - no shim needed. +Declare both in `debian/control`: + +``` +Build-Driver: debmagic +Build-Depends: debmagic-pkg, debmagic-dpkg-driver +``` + +`debmagic-pkg` provides the python API that `rules.py` imports, `debmagic-dpkg-driver` the perl module dpkg loads. +`dpkg-buildpackage` (≥ 1.23.6) then loads the `Dpkg::BuildDriver::Debmagic` perl module, which dispatches the dpkg targets (`clean`, `build`, `binary`, ...) to `debian/rules.py`. +Root handling (`Rules-Requires-Root`, fakeroot) works exactly as with `debian/rules` Makefiles. diff --git a/docs/usage/source.md b/docs/usage/source.md index d1907e14..1ed4e142 100644 --- a/docs/usage/source.md +++ b/docs/usage/source.md @@ -23,14 +23,29 @@ debmagic build source --source-dir /path/to/parent/of/debian/dir --output-dir /p ## Uploading to Launchpad ```shell +# build, sign, then upload in one go: debmagic build source --sign --sign-key you@example.com \ - --source-dir . --output-dir /tmp/out -dput ppa:your-lp-username/your-ppa /tmp/out/*_source.changes + --upload ppa:your-lp-username/your-ppa + +# or upload an already-built .changes separately: +debmagic upload ppa:your-lp-username/your-ppa ./build/pkg_*_source.changes ``` - `--sign` GPG-signs the `.dsc`/`.buildinfo`/`.changes` after building, on the host with your gpg keyring — see [Signing](build.md#signing). - `--sign-key` picks which key/uid to sign with; omit it to fall back to the `Changed-By:`/`Maintainer:` address of the file being signed. -- Both can be set as defaults in `debian/debmagic.toml`/`$XDG_CONFIG_HOME/debmagic/config.toml` instead of passing them every time: +- `--upload` picks the upload target (`name` or `name:parameter`), resolved from `[upload.targets]` over the builtins (`ppa`, `ubuntu`, `debian`) — see [Uploading](upload.md). Available on binary builds too. +- `--include-orig=auto|yes|no` controls whether the built `.changes` references the `orig` tarball (`-sa`/`-sd`): `auto` (default) includes it only when the archive cannot have it yet — a new upstream version or a deltarebase onto Debian. See [Upstream versions](upstream.md#including-the-orig-in-uploads). +- `--changes-option` adds an extra field to the `.changes` file, passed to `dpkg-buildpackage` as-is (repeatable). This is what git-ubuntu's `prepare-upload` uses to correlate source uploads with git history: + + ```shell + debmagic build source --sign --sign-key you@example.com \ + --changes-option=-DVcs-Git=https://git.launchpad.net/~you/ubuntu/+source/pkg \ + --changes-option=-DVcs-Git-Ref=refs/heads/lp12345-fix \ + --changes-option=-DVcs-Git-Commit=36349e3669439603206cffc01dcb903b0ba2c134 \ + --upload ppa:you/your-ppa + ``` + +- Both signing settings and upload targets can be set as defaults in `debian/debmagic.toml`/`$XDG_CONFIG_HOME/debmagic/config.toml` instead of passing them every time: ```toml [sign] @@ -44,6 +59,9 @@ The same file selection as `debmagic build` uses to populate the build environme Untracked-but-not-ignored files are included, so uncommitted work-in-progress changes are packaged too — useful while iterating locally. `debian/source/options` (`tar-ignore`/`diff-ignore` patterns, etc.) is honored as usual, since it's `dpkg-source` itself that reads it. +Debian packages also require their `orig` tarball (the "upstream" sources). +For `3.0 (quilt)` packages the `orig` tarball is located automatically — see [Upstream versions](upstream.md#orig-tarballs-in-builds). + ## What's NOT run By default, `debian/rules` is never invoked (neither `dpkg-source` nor `dpkg-genchanges` need it), so this also works for source trees whose build-dependencies aren't installed anywhere. diff --git a/docs/usage/test.md b/docs/usage/test.md index 9297f69f..96393f17 100644 --- a/docs/usage/test.md +++ b/docs/usage/test.md @@ -33,6 +33,7 @@ The driver *is* the testbed: `autopkgtest` runs with the `null` backend inside t | `--distro ` | Override the target distro for the test environment (defaults to the prior build's distro from `environment.json`, not the changelog) | | `--proposed` | Enable the `-proposed` pocket in the test environment | | `--apt-mirror ` | Mirror URL (same as [`debmagic build`](build.md)) | +| `--apt-update-age ` | Configure when to run `apt-get update` (same as [`debmagic build`](build.md#apt-update-age)) | | `--source-dir ` | Directory containing the `debian/` package directory | | `--allow-host-test` | Allow the bare driver, which runs autopkgtest as root on the host | | `--shell-on-failure` | On test failure, drop into an interactive shell in the test environment when stdout is a TTY | diff --git a/docs/usage/upload.md b/docs/usage/upload.md new file mode 100644 index 00000000..497b5f19 --- /dev/null +++ b/docs/usage/upload.md @@ -0,0 +1,107 @@ +# Uploading + +`debmagic upload` uploads a signed `.changes` file — and every file it references (`.dsc`, tarballs, `.buildinfo`) — to an archive or PPA, like `dput`. +It always uses ssh-based transports (`scp`/`sftp`), reusing your `~/.ssh/config` (host keys, agents, proxies, users) like any other ssh tool. + +## TL;DR + +```shell +# upload to your launchpad ppa (needs a signed .changes in the output dir): +debmagic upload ppa:your-lp-username/your-ppa + +# or build, sign and upload in one go: +debmagic build source --sign --sign-key you@example.com --upload ppa:your-lp-username/your-ppa + +# upload a specific .changes file to a host defined in debmagic.toml: +debmagic upload myhost ./build/pkg_1.0-1_source.changes +``` + +## Upload targets + +The first argument is the *target*: `name` or `name:parameter` (split on the first `:`). + +Targets resolve in this order: + +1. `[upload.targets.]` in `debmagic.toml`, merged field-by-field over a same-named builtin (so a config can override just e.g. `incoming` and keep the rest) +2. built-in targets: + +| name | server | incoming | login | method | +|---|---|---|---|---| +| `ppa` | `ppa.launchpad.net` | `~{target}/ubuntu` | `anonymous` | sftp | +| `ubuntu` | `upload.ubuntu.com` | `ubuntu` | `anonymous` | sftp | +| `debian` | `ssh.upload.debian.org` | `/srv/upload.debian.org/UploadQueue` | ssh config | sftp | + +The `:parameter` fills the `{target}` placeholder of the target's `incoming`/`server` — for `ppa:your-lp-username/your-ppa` the incoming dir becomes `~your-lp-username/your-ppa/ubuntu`. +Both PPAs and the Ubuntu archive accept sftp uploads. + +A target with no builtin of that name is an error unless it's fully configured in `debmgic.toml`: + +```toml +[upload.targets.myhost] +method = "sftp" # or "scp"; unset keeps a builtin's method +server = "example.com" +incoming = "/srv/incoming" +login = "sfttech" # optional; unset lets the ssh config decide +port = 2222 # optional; unset lets the ssh config decide +pre_upload_commands = [ + "lintian {changes}", +] +``` + +## Pre-upload checks + +`pre_upload_commands` run *before anything is transferred*, each via `sh -c`: + +- the `{changes}` placeholder is substituted with the `.changes` file path +- `DEBMAGIC_UPLOAD_CHANGES`, `DEBMAGIC_UPLOAD_TARGET`, `DEBMAGIC_UPLOAD_TARGET_SERVER` and `DEBMAGIC_UPLOAD_TARGET_INCOMING` are set in the environment +- a non-zero exit aborts the upload entirely + +This is deliberately simple: it's a list of commands, not a plugin system. +Existing `dput-ng` hooks (which use its python "api") can be bridged by wrapping them in one command later, and debmagic's own built-in linter will simply be called from here too. + +Pass `--no-hooks` to skip the checks. + +## Including the `orig` tarball + +`--include-orig=auto|yes|no` decides whether the upload carries the `orig` tarball (the `-sa`/`-sd` choice, made at upload time): + +- `auto` (default): include only when the archive provably lacks this upstream version's orig (upstream version bump, deltarebase) +- `yes`/`no`: always/never + +When the decision disagrees with the `.changes` file, its file listing is rewritten (orig entry added or removed, checksums recomputed) and it is re-signed. +Before adding, the tarball is verified against the checksum the `.dsc` recorded — a stale orig fails loudly instead of being rejected by the archive. +See [Upstream versions](upstream.md) for the full orig lifecycle. + +## Upload log and `--force` + +Every successful upload is recorded in a structured JSON file next to the `.changes`, e.g. `pkg_1.0-1_source.upload.json`: + +```json +{ + "uploads": [ + { "target": "ppa:your-lp-username/your-ppa", "time": "2026-09-18T15:16:02+02:00" } + ] +} +``` + +Uploading the same `.changes` to a target that already has a successful upload recorded is refused: + +```shell +debmagic upload ppa:your-lp-username/your-ppa # "already has a successful upload ..." +debmagic upload --force ppa:your-lp-username/your-ppa # uploads anyway +``` + +The check is per target spec (including the `:parameter`), so uploading the same version to several PPAs works without `--force`. + +## Upload methods + +| method | how | +|---|---| +| `scp` | `scp -p [-P port] :/` | +| `sftp` | `sftp -b -` batch mode, `put`ting each file into `incoming` | + +Both run over ssh and read `~/.ssh/config` themselves (user, port, keys, proxies), so only explicit `login`/`port` target settings are passed on the command line. + +sftp paths are relative to the login's home on the server (`ubuntu` for the Ubuntu archive, `~your-lp-username/your-ppa/ubuntu` for PPAs), which is why the builtin `incoming` values above look like they do. + +The methods share the [`Uploader`](https://debmagic.readthedocs.io) trait; a new method (e.g. git-based uploads) is a new config value plus one implementation. diff --git a/docs/usage/upstream.md b/docs/usage/upstream.md new file mode 100644 index 00000000..914cca64 --- /dev/null +++ b/docs/usage/upstream.md @@ -0,0 +1,114 @@ +# Upstream versions + +`debmagic upstream` replaces `uscan`/`uupdate`: it reads the package's existing `debian/watch` and `debian/copyright` and offers two operations — checking for new upstream versions and switching the package tree to one. + +```shell +debmagic upstream list # newest upstream version newer than the changelog's +debmagic upstream list --all # every candidate, newest first +debmagic upstream list --previous 3 # the 3 newest versions above the current one +debmagic upstream switch latest # switch the tree to the newest version +debmagic upstream switch 3.12.0 # switch to a specific version +debmagic upstream switch latest --dry-run +``` + +## `upstream list` + +Queries the sources declared in `debian/watch` and prints candidate upstream versions, sorted with Debian version semantics. +By default only versions newer than the changelog's current upstream version are shown (newest first); `--all` shows everything the watch file finds. + +The changelog's version is normalized with the watch file's `Dversion-Mangle` rules before comparing, so `+dfsg` suffixes and similar don't hide available updates. + +## `upstream switch` + +Fetches the chosen version's tarball, applies the package's repack configuration, writes the `orig` tarball into the package's output dir (`build/` by default) and replaces the source tree's contents — everything except `debian/`, which is kept as-is. +The changelog is not touched; add the new version entry yourself (automated changelog handling is planned). + +`--dry-run` reports what would happen — tarball URL, repack excludes, version suffix — without downloading anything. + +For a concrete version, the tarball URL is resolved without scraping the download listing: + +1. the distro archives (the `launchpad` orig method) — a version Debian or Ubuntu published needs no upstream download at all +2. the watch pattern, read as a URL template: `@ANY_VERSION@` becomes the version, `@ARCHIVE_EXT@` each archive extension in turn (the extension of the project's existing orig tarball is probed first), and the constructed URLs are probed — this also finds versions that have already fallen out of the listing +3. the listing, fetched as a *structure teacher*: any listed sibling reveals the prefix/extension shape, and the requested version's URL is constructed from it — this covers patterns pure inversion rejects (multiple capture groups, regex constructs) and compression changes +4. the listing, filtered for the requested version — the last resort + +`upstream list` always scrapes the listing, since enumeration is its job. + +### Signature verification + +After downloading, the tarball's PGP signature is verified against the keyring in `debian/upstream/signing-key.asc` — the same location on every dpkg distribution, Ubuntu included (Ubuntu packages use the same `debian/` packaging layout; there is no Ubuntu-specific path). +Verification is on by default and uses the same OpenPGP backend as signing — `gpg` by default, `sq` (Sequoia) when `sign.tool = "sequoia"`, or your custom command when `sign.tool = "custom"`: `sign.verify_command` with `{file}`, `{signature}` and `{keyring}` placeholders (without `{signature}` the signature path is appended as the last argument), falling back to `sign.sign_command`. + +The signature is looked up via the watch file's `Pgp-Sig-Url-Mangle` rules; without them the common suffixes (`.asc`, `.sig`, `.sign`, `.pgp`, `.gpg`) are tried next to the tarball. +If no signature exists upstream, the switch fails — disable the check per-run with `--no-signature-check` or persistently with `verify_signatures = false`: + +```toml +[upstream] +verify_signatures = false +``` + +Export the upstream key with `gpg --export --armor > debian/upstream/signing-key.asc`. + +### Repacking + +Two sources are merged: + +- `Files-Excluded` in `debian/copyright` — the standard way to strip non-DFSG or useless files; `Files-Excluded-` works for multiple-upstream-tarball packages +- the watch file's `Repack`/`Repacksuffix` options, e.g. `Repacksuffix: +dfsg` appends the suffix to the upstream version of the repacked tarball + +The repacked `orig` tarball uses the canonical Debian layout: a single top-level `_/` directory, xz-compressed. + +## Watch file support + +`debian/watch` versions 2–5 are parsed (version 1 is rejected). +The common options work: `Source`, `Matching-Pattern` (including the `@ANY_VERSION@`/`@ARCHIVE_EXT@`/`@PACKAGE@` substitutions), `Search-Mode`, `Uversion-Mangle`, `Dversion-Mangle`, `Filename-Mangle`, `Download-Url-Mangle`, `Pgp-Mode`/`Pgp-Sig-Url-Mangle`, `Repack`, `Repacksuffix`, `Component` (for MUT packages), `Untrackable`. + +http(s) sources are fetched directly; ftp sources fall back to `curl` (install it when a watch file needs it). + +Rare options with no debmagic equivalent yet (`Mode: git`/`svn`, `Version-Schema: group/checksum`, `Page-Mangle`, `Update-Script`, ...) produce a clear error suggesting an explicit upstream declaration instead of silently misbehaving. + +## Orig tarballs in builds + +`debmagic build source` needs the `orig` tarball for `3.0 (quilt)` packages and finds it without any configuration: + +1. the package's output dir (`build/` by default) — where `upstream switch` puts it +2. next to the source tree (`../`), the conventional location — used as-is, never written to +3. the download cache (`~/.cache/debmagic/orig///`) — populated by the `launchpad` method; a cached tarball is hardlinked into the output dir when both are on the same filesystem, copied otherwise +4. the configured fetch method — `launchpad` by default, no configuration needed: + +```toml +[orig_tarball] +method = "launchpad" # the default: the distro archives via Launchpad's download URLs +# or: +method = "debian" # Debian's own archive pool, without Launchpad +# method = "ubuntu" # Ubuntu's own archive pool, without Launchpad +# mirrors default to the distro archives; any apt-style mirror root works: +# debian_mirror = "https://mirror.example.com/debian" +# ubuntu_mirror = "http://mirror.example.com/ubuntu" +# or: +method = "custom" # requires 'command' +command = "uscan --download --download-current-version && cp ../foo*.orig.tar.* {output_dir}/" +# or: +method = "disabled" # never fetch; build only with tarballs found locally +``` + +The `launchpad` method constructs the orig tarball's download URL directly (`launchpad.net//+archive/primary/+files/_.orig.tar.`) and probes the compression extensions, searching Ubuntu first, then Debian (which Launchpad mirrors). No API query, configuration or authentication is needed — and no published version of the changelog head either: the orig tarball only depends on the upstream part, which older publications share. + +The custom command runs via `sh -c` in the source dir with `{name}`, `{version}`, `{upstream_version}`, `{source_dir}` and `{output_dir}` placeholders, and `DEBMAGIC_ORIG_OUTPUT_DIR` set. + +`3.0 (native)` packages skip all of this — they have no `orig` tarball. + +## Including the orig in uploads + +Whether an upload carries the `orig` tarball is decided at upload time, not build time: + +```shell +debmagic build source --include-orig=auto # default +debmagic upload ppa:you/your-ppa --include-orig=yes +``` + +- `auto` (default): include when the target hasn't seen this upstream version yet — no upload record, a different recorded version, or a deltarebase onto a Debian version Ubuntu never had (detected from the changelog: current revision has an `ubuntu` component, the previous entry's doesn't) +- `yes`/`no`: always/never + +On `upload`, a decision that disagrees with the `.changes` file rewrites its file listing (adding or removing the `orig` entry with correct checksums) and re-signs it. +Before adding, the tarball is verified against the checksum the `.dsc` recorded, so a stale orig next to the `.changes` fails loudly instead of being rejected by the archive. \ No newline at end of file diff --git a/justfile b/justfile index 1c6e3659..dce7a4f3 100644 --- a/justfile +++ b/justfile @@ -44,6 +44,15 @@ ci: fmt-check lint typecheck test docs pre-commit: uv run pre-commit run --all-files +# Regenerate the debmagic(1) manpage from the CLI's --help output +manpage: + cargo build -p debmagic + help2man --no-info -N -o debian/debmagic.1 target/debug/debmagic + +# Build the debmagic snap +snap: + snapcraft pack + # Build debmagic itself with the docker driver self-build *args: cargo run --locked -p debmagic -- build binary --driver=docker --persistent --incremental {{ args }} diff --git a/packages/debmagic-common/Cargo.toml b/packages/debmagic-common/Cargo.toml index ac3a451b..ecec723c 100644 --- a/packages/debmagic-common/Cargo.toml +++ b/packages/debmagic-common/Cargo.toml @@ -10,9 +10,15 @@ edition.workspace = true rust-version.workspace = true [dependencies] +anyhow = { workspace = true } chrono = { workspace = true } +deb822-lossless = { workspace = true } +debian-changelog = { workspace = true } +md-5 = { workspace = true } regex = { workspace = true } serde = { workspace = true, features = ["derive"] } +sha1 = { workspace = true } +sha2 = { workspace = true } [dev-dependencies] test-case = { workspace = true } diff --git a/packages/debmagic-common/src/changes.rs b/packages/debmagic-common/src/changes.rs new file mode 100644 index 00000000..dbeaf940 --- /dev/null +++ b/packages/debmagic-common/src/changes.rs @@ -0,0 +1,234 @@ +use std::path::{Path, PathBuf}; + +use deb822_lossless::Paragraph; + +use crate::debian::control::Hash; + +/// A parsed `.changes` file listing: which files it references and +/// whether the `orig` tarball(s) are among them. +#[derive(Debug, Clone, Default)] +pub struct ChangesFiles { + /// All filenames listed under `Files:`/`Checksums-*:`. + pub files: Vec, +} + +impl ChangesFiles { + /// The `orig` tarball entries: `_.orig.tar.` + /// and component origs `_.orig-.tar.`. + pub fn orig_tarballs(&self) -> Vec<&str> { + self.files + .iter() + .map(String::as_str) + .filter(|name| is_orig_tarball(name)) + .collect() + } +} + +/// Whether a filename is an `orig` tarball (main or component). +pub fn is_orig_tarball(name: &str) -> bool { + // main: foo_1.0.orig.tar.gz; component: foo_1.0.orig-bar.tar.gz + let Some((_, rest)) = name.split_once(".orig") else { + return false; + }; + if rest.starts_with(".tar.") { + return true; + } + // component: the part between `.orig-` and `.tar.` is the component name + match rest.strip_prefix('-').and_then(|r| r.split_once(".tar.")) { + Some((component, _)) => !component.is_empty() && !component.contains('/'), + None => false, + } +} + +/// The `orig` tarball filename prefix for a source package version: +/// `_.orig.tar.` — the compression extension varies. +/// For a component, `_.orig-.tar.`. +pub fn orig_prefix(name: &str, upstream_version: &str) -> String { + format!("{name}_{upstream_version}.orig.tar.") +} + +/// The `orig` tarball filename prefix for a component of a source +/// package version: `_.orig-.tar.`. +pub fn component_orig_prefix(name: &str, upstream_version: &str, component: &str) -> String { + format!("{name}_{upstream_version}.orig-{component}.tar.") +} + +/// Locate an existing `orig` tarball in `dir`, trying the extensions +/// dpkg-source accepts. Component origs (`orig-`) are not +/// matched; use `find_component_in_dir` for those. +pub fn find_orig_in_dir(dir: &Path, name: &str, upstream_version: &str) -> Option { + let prefix = orig_prefix(name, upstream_version); + for ext in ["gz", "xz", "bz2", "lzma"] { + let candidate = dir.join(format!("{prefix}{ext}")); + if candidate.is_file() { + return Some(candidate); + } + } + None +} + +/// Locate a component `orig` tarball (`_.orig-.tar.`) +/// in `dir`. +pub fn find_component_in_dir( + dir: &Path, + name: &str, + upstream_version: &str, + component: &str, +) -> Option { + let prefix = component_orig_prefix(name, upstream_version, component); + for ext in ["gz", "xz", "bz2", "lzma"] { + let candidate = dir.join(format!("{prefix}{ext}")); + if candidate.is_file() { + return Some(candidate); + } + } + None +} + +/// Add one `orig` tarball entry (or, with `None`, remove all orig +/// entries) from a `.changes` paragraph's `Files:`/`Checksums-*:` +/// listings. `tarball_data` is the content of the tarball named by +/// `orig_name`, used for the checksum and size entries; it is unused +/// when removing. Returns whether the paragraph was modified. +pub fn set_changes_orig_entries( + control: &mut Paragraph, + orig_name: Option<&str>, + tarball_data: &[u8], +) -> bool { + let mut modified = false; + + let keys: Vec = control.keys().collect(); + for key in keys { + if key != "Files" && !key.starts_with("Checksums-") { + continue; + } + let Some(value) = control.get(&key) else { + continue; + }; + let mut lines: Vec = Vec::new(); + for line in value.lines().filter(|l| !l.is_empty()) { + let name = line.split_whitespace().next_back().unwrap_or_default(); + let is_orig = is_orig_tarball(name); + match orig_name { + // adding: keep every existing entry, origs included + Some(_) => lines.push(line.to_string()), + // removing: drop all orig entries + None if is_orig => modified = true, + None => lines.push(line.to_string()), + } + } + if let Some(name) = orig_name { + let already = lines + .iter() + .any(|l| l.split_whitespace().next_back() == Some(name)); + if !already { + modified = true; + let hash = match key.as_str() { + "Files" => Hash::Md5.hex(tarball_data), + "Checksums-Sha1" => Hash::Sha1.hex(tarball_data), + "Checksums-Sha256" => Hash::Sha256.hex(tarball_data), + _ => continue, + }; + lines.push(format!(" {} {} {}", hash, tarball_data.len(), name)); + } + } + control.set(&key, &lines.join("\n")); + } + + modified +} + +#[cfg(test)] +mod tests { + use super::*; + use test_case::test_case; + + #[test_case("pkg_1.0.orig.tar.gz", true; "main orig")] + #[test_case("pkg_1.0.orig.tar.xz", true; "main orig xz")] + #[test_case("pkg_1.0.orig-bar.tar.xz", true; "component orig")] + #[test_case("pkg_1.0-1.debian.tar.xz", false; "debian tarball")] + #[test_case("pkg_1.0-1.dsc", false; "dsc")] + #[test_case("pkg_1.0.orig.tar", false; "no compression ext")] + fn test_is_orig_tarball(name: &str, expected: bool) { + assert_eq!(is_orig_tarball(name), expected); + } + + #[test] + fn test_changes_files_orig_tarballs() { + let changes = ChangesFiles { + files: vec![ + "pkg_1.0.orig.tar.xz".to_string(), + "pkg_1.0.orig-bar.tar.xz".to_string(), + "pkg_1.0-1.debian.tar.xz".to_string(), + "pkg_1.0-1.dsc".to_string(), + ], + }; + assert_eq!( + changes.orig_tarballs(), + vec!["pkg_1.0.orig.tar.xz", "pkg_1.0.orig-bar.tar.xz"] + ); + } + + #[test] + fn test_find_orig_in_dir() { + let dir = std::env::temp_dir().join(format!("debmagic-common-orig-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + assert_eq!(find_orig_in_dir(&dir, "pkg", "1.0"), None); + std::fs::write(dir.join("pkg_1.0.orig.tar.xz"), "x").unwrap(); + assert_eq!( + find_orig_in_dir(&dir, "pkg", "1.0"), + Some(dir.join("pkg_1.0.orig.tar.xz")) + ); + // a component orig is not picked up by the main lookup + std::fs::write(dir.join("pkg_1.0.orig-bar.tar.xz"), "x").unwrap(); + assert_eq!( + find_orig_in_dir(&dir, "pkg", "1.0"), + Some(dir.join("pkg_1.0.orig.tar.xz")) + ); + assert_eq!( + find_component_in_dir(&dir, "pkg", "1.0", "bar"), + Some(dir.join("pkg_1.0.orig-bar.tar.xz")) + ); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn set_changes_orig_entries_adds_and_removes() { + let mut control: Paragraph = "Format: 1.8\nSource: pkg\nFiles:\n abc 3 pkg_1.0-1.debian.tar.xz\n def 4 pkg_1.0-1.dsc\nChecksums-Sha256:\n xyz 3 pkg_1.0-1.debian.tar.xz\n zzz 4 pkg_1.0-1.dsc\n" + .parse() + .unwrap(); + + // add + assert!(set_changes_orig_entries( + &mut control, + Some("pkg_1.0.orig.tar.xz"), + b"tarball" + )); + let content = control.to_string(); + assert!(content.contains("pkg_1.0.orig.tar.xz")); + assert_eq!(content.matches("pkg_1.0.orig.tar.xz").count(), 2); + + // adding again is a no-op + assert!(!set_changes_orig_entries( + &mut control, + Some("pkg_1.0.orig.tar.xz"), + b"tarball" + )); + + // adding a component orig keeps the main one + assert!(set_changes_orig_entries( + &mut control, + Some("pkg_1.0.orig-bar.tar.xz"), + b"component" + )); + let content = control.to_string(); + assert!(content.contains("pkg_1.0.orig.tar.xz")); + assert!(content.contains("pkg_1.0.orig-bar.tar.xz")); + + // remove drops all origs, main and component + assert!(set_changes_orig_entries(&mut control, None, b"")); + let content = control.to_string(); + assert!(!content.contains("orig.tar")); + assert!(!content.contains("orig-bar")); + } +} diff --git a/packages/debmagic-common/src/debian.rs b/packages/debmagic-common/src/debian.rs index a6db76ad..ad01639e 100644 --- a/packages/debmagic-common/src/debian.rs +++ b/packages/debmagic-common/src/debian.rs @@ -1 +1,5 @@ +pub mod changelog; +pub mod control; +pub mod copyright; +pub mod source; pub mod version; diff --git a/packages/debmagic-common/src/debian/changelog.rs b/packages/debmagic-common/src/debian/changelog.rs new file mode 100644 index 00000000..96f90c7b --- /dev/null +++ b/packages/debmagic-common/src/debian/changelog.rs @@ -0,0 +1,162 @@ +use crate::debian::version::PackageVersion; + +/// The head entry of a `debian/changelog`: what identifies the +/// package and where the upload is headed. +#[derive(Debug, Clone, PartialEq)] +pub struct ChangelogHead { + pub package: String, + pub version: PackageVersion, + pub distributions: Vec, +} + +impl ChangelogHead { + /// Extract the head entry of a parsed changelog. + pub fn from_changelog(changelog: &debian_changelog::ChangeLog) -> Option { + let first = changelog.iter().next()?; + Some(Self { + package: first.package()?, + version: PackageVersion::new( + first.version()?.epoch, + first.version()?.upstream_version, + first.version()?.debian_revision, + ), + distributions: first.distributions()?, + }) + } +} + +/// Whether a changelog's top entry looks like an Ubuntu deltarebase +/// onto a Debian version Ubuntu has never seen: the current revision +/// carries an `ubuntu` component while the previous entry's does not. +/// In that case the Ubuntu archive cannot have the orig tarball yet. +pub fn is_deltarebase_onto_debian(changelog: &debian_changelog::ChangeLog) -> bool { + let mut entries = changelog.iter().take(2); + let Some(current) = entries.next() else { + return false; + }; + let Some(previous) = entries.next() else { + // a single entry says nothing about a rebase; the upload + // record decides whether the target needs the orig tarball + return false; + }; + let Some(current_version) = current.version() else { + return false; + }; + let Some(previous_version) = previous.version() else { + return false; + }; + let current_is_ubuntu = revision_is_ubuntu(current_version.debian_revision.as_deref()); + let previous_is_ubuntu = revision_is_ubuntu(previous_version.debian_revision.as_deref()); + current_is_ubuntu && !previous_is_ubuntu +} + +/// Whether a changelog's top entry bumps the upstream part relative +/// to the entry below: the orig tarball of the new upstream version +/// cannot be in the archive yet, so an upload must carry it. +pub fn is_new_upstream_version(changelog: &debian_changelog::ChangeLog) -> bool { + let mut entries = changelog.iter().take(2); + let Some(current) = entries.next() else { + return false; + }; + let Some(previous) = entries.next() else { + // a single entry says nothing about what the archive has + return false; + }; + let (Some(current), Some(previous)) = (current.version(), previous.version()) else { + return false; + }; + current.upstream_version != previous.upstream_version +} + +/// A Debian revision contains an `ubuntu` component, e.g. `1ubuntu2`. +fn revision_is_ubuntu(revision: Option<&str>) -> bool { + revision.is_some_and(|r| r.contains("ubuntu")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(changelog: &str) -> debian_changelog::ChangeLog { + changelog.parse().unwrap() + } + + fn entry(version: &str) -> String { + format!( + "foo ({version}) unstable; urgency=medium\n\n * Some change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n" + ) + } + + #[test] + fn test_deltarebase_detected() { + let changelog = parse(&format!( + "{}\n{}", + entry("1.2.3-1ubuntu1"), + entry("1.2.3-1") + )); + assert!(is_deltarebase_onto_debian(&changelog)); + } + + #[test] + fn test_consecutive_ubuntu_uploads_not_deltarebase() { + let changelog = parse(&format!( + "{}\n{}", + entry("1.2.3-1ubuntu2"), + entry("1.2.3-1ubuntu1") + )); + assert!(!is_deltarebase_onto_debian(&changelog)); + } + + #[test] + fn test_pure_debian_not_deltarebase() { + let changelog = parse(&format!("{}\n{}", entry("1.2.3-2"), entry("1.2.3-1"))); + assert!(!is_deltarebase_onto_debian(&changelog)); + } + + #[test] + fn test_single_entry_not_deltarebase() { + let changelog = parse(&entry("1.2.3-1ubuntu1")); + assert!(!is_deltarebase_onto_debian(&changelog)); + } + + #[test] + fn test_new_upstream_version_detected() { + let changelog = parse(&format!( + "{}\n{}", + entry("1.2.3-1ubuntu1"), + entry("1.2.2-3ubuntu4") + )); + assert!(is_new_upstream_version(&changelog)); + } + + #[test] + fn test_same_upstream_not_new_upstream() { + let changelog = parse(&format!( + "{}\n{}", + entry("1.2.3-1ubuntu2"), + entry("1.2.3-1ubuntu1") + )); + assert!(!is_new_upstream_version(&changelog)); + } + + #[test] + fn test_new_upstream_single_entry() { + let changelog = parse(&entry("1.2.3-1")); + assert!(!is_new_upstream_version(&changelog)); + } + + #[test] + fn test_head_from_changelog() { + let changelog = parse(&entry("1.2.3-1ubuntu2")); + let head = ChangelogHead::from_changelog(&changelog).unwrap(); + assert_eq!(head.package, "foo"); + assert_eq!(head.version.version(), "1.2.3-1ubuntu2"); + assert_eq!(head.distributions, ["unstable"]); + } + + #[test] + fn test_head_from_empty_changelog() { + let changelog = parse(""); + assert!(ChangelogHead::from_changelog(&changelog).is_none()); + } +} diff --git a/packages/debmagic-common/src/debian/control.rs b/packages/debmagic-common/src/debian/control.rs new file mode 100644 index 00000000..b193bb17 --- /dev/null +++ b/packages/debmagic-common/src/debian/control.rs @@ -0,0 +1,229 @@ +use deb822_lossless::Paragraph; + +/// Filenames ending in `.` listed under `Files:` or any +/// `Checksums-*:` field. +pub fn child_filename(paragraph: &Paragraph, ext: &str) -> Option { + let suffix = format!(".{ext}"); + for key in paragraph.keys() { + if key != "Files" && !key.starts_with("Checksums-") { + continue; + } + for line in paragraph + .get(&key) + .into_iter() + .flat_map(|v| v.lines().map(str::to_string).collect::>()) + { + if let Some(name) = line.split_whitespace().next_back() + && name.ends_with(&suffix) + { + return Some(name.to_string()); + } + } + } + None +} + +/// The hash algorithms used by the checksum fields of a control file. +#[derive(Clone, Copy)] +pub enum Hash { + Md5, + Sha1, + Sha256, +} + +impl Hash { + pub fn hex(self, data: &[u8]) -> String { + fn hex(mut hasher: D, data: &[u8]) -> String { + hasher.update(data); + hasher + .finalize() + .iter() + .map(|b| format!("{b:02x}")) + .collect() + } + match self { + Hash::Md5 => hex(md5::Md5::default(), data), + Hash::Sha1 => hex(sha1::Sha1::default(), data), + Hash::Sha256 => hex(sha2::Sha256::default(), data), + } + } +} + +/// The checksum fields debmagic understands, mapped to their hash. +pub const CHECKSUM_FIELDS: &[(&str, Hash)] = &[ + ("Files", Hash::Md5), + ("Checksums-Sha1", Hash::Sha1), + ("Checksums-Sha256", Hash::Sha256), +]; + +/// Rewrite one `Files:`/`Checksums-*:` line: the first token is the +/// checksum, the second the size, the last the filename; entries for +/// other files pass through unchanged. +fn rewrite_checksum_line(line: &str, filename: &str, checksum: &str, size: usize) -> String { + let tokens: Vec<&str> = line.split_whitespace().collect(); + match tokens.as_slice() { + [old_checksum, old_size, middle @ .., name] if *name == filename => { + let middle = if middle.is_empty() { + String::new() + } else { + format!(" {}", middle.join(" ")) + }; + format!("{checksum} {size}{middle} {name}") + } + _ => line.to_string(), + } +} + +/// Rewrite the size and checksum entries for the file listings in a control file +pub fn fixup_checksums( + paragraph: &mut Paragraph, + filename: &str, + data: &[u8], +) -> anyhow::Result<()> { + let size = data.len(); + + for key in paragraph.keys() { + if key.starts_with("Checksums-") && !CHECKSUM_FIELDS.iter().any(|(field, ..)| *field == key) + { + // An unknown checksum format would keep a stale checksum for a + // re-signed file, producing an upload that fails verification + // far away from here. + anyhow::bail!("unknown checksum field '{key}:' in control file"); + } + } + + for (key, hash) in CHECKSUM_FIELDS { + let Some(value) = paragraph.get(key) else { + continue; + }; + let checksum = hash.hex(data); + let updated = value + .lines() + .filter(|line| !line.is_empty()) + .map(|line| rewrite_checksum_line(line, filename, &checksum, size)) + .collect::>() + .join("\n"); + paragraph.set(key, &updated); + } + Ok(()) +} + +/// The digests of one file, for every algorithm a control file's +/// checksum fields may use. +#[derive(Debug, Clone, Default)] +pub struct Digests { + pub md5: String, + pub sha1: String, + pub sha256: String, +} + +impl Digests { + /// Digest `data` with every checksum-field algorithm at once. + pub fn of(data: &[u8]) -> Self { + Self { + md5: Hash::Md5.hex(data), + sha1: Hash::Sha1.hex(data), + sha256: Hash::Sha256.hex(data), + } + } + + fn for_hash(&self, hash: Hash) -> &str { + match hash { + Hash::Md5 => &self.md5, + Hash::Sha1 => &self.sha1, + Hash::Sha256 => &self.sha256, + } + } +} + +/// Verify that the checksum entries a paragraph records for `filename` +/// match `digests`. Returns whether any entry was checked; a paragraph +/// listing no entry for the file is not an error. +pub fn verify_checksums( + paragraph: &Paragraph, + filename: &str, + digests: &Digests, +) -> anyhow::Result { + let mut checked = false; + for (field, hash) in CHECKSUM_FIELDS { + let Some(value) = paragraph.get(field) else { + continue; + }; + let actual = digests.for_hash(*hash); + for line in value.lines().filter(|l| !l.is_empty()) { + let tokens: Vec<&str> = line.split_whitespace().collect(); + if tokens.last() != Some(&filename) { + continue; + } + let Some(&expected) = tokens.first() else { + continue; + }; + checked = true; + if actual != expected { + anyhow::bail!("{filename} does not match the checksum {field} recorded for it"); + } + } + } + Ok(checked) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn verify_checksums_matches_and_mismatches() { + let data = b"tarball content"; + let digests = Digests::of(data); + let control: Paragraph = format!( + "Format: 3.0 (quilt)\nFiles:\n {} 15 debiandev optional pkg_1.0.orig.tar.xz\nChecksums-Sha256:\n {} 15 pkg_1.0.orig.tar.xz\n", + digests.md5, digests.sha256 + ) + .parse() + .unwrap(); + + assert!(verify_checksums(&control, "pkg_1.0.orig.tar.xz", &digests).unwrap()); + + // a stale tarball fails + let stale = Digests::of(b"different content"); + assert!(verify_checksums(&control, "pkg_1.0.orig.tar.xz", &stale).is_err()); + + // a file the paragraph does not list is not checked + assert!(!verify_checksums(&control, "other.tar.xz", &digests).unwrap()); + } + + fn parse_control(content: &str) -> Paragraph { + content.parse().unwrap() + } + + #[test] + fn child_filename_finds_dsc_and_buildinfo() { + let control = parse_control( + "Format: 1.8\nSource: pkg\nFiles:\n abc 123 pkg_1.0.dsc\n def 456 pkg_1.0.buildinfo\n ghi 789 other.txt\nChecksums-Sha256:\n xyz 123 pkg_1.0.dsc\n", + ); + assert_eq!( + child_filename(&control, "dsc").as_deref(), + Some("pkg_1.0.dsc") + ); + assert_eq!( + child_filename(&control, "buildinfo").as_deref(), + Some("pkg_1.0.buildinfo") + ); + assert_eq!(child_filename(&control, "deb"), None); + } + + #[test] + fn fixup_rewrites_all_checksum_sections() { + let mut control = parse_control( + "Format: 1.8\nFiles:\n oldmd5 3 hash optional pkg_1.0.dsc\nChecksums-Sha1:\n oldsha1 3 pkg_1.0.dsc\nChecksums-Sha256:\n oldsha256 3 pkg_1.0.dsc\n", + ); + let data = b"abc"; + fixup_checksums(&mut control, "pkg_1.0.dsc", data).unwrap(); + + let md5 = Hash::Md5.hex(data); + let joined = control.to_string(); + assert!(joined.contains(&format!("{md5} 3 hash optional pkg_1.0.dsc"))); + assert!(joined.contains(&format!(" {} 3 pkg_1.0.dsc", Hash::Sha1.hex(data)))); + assert!(joined.contains(&format!(" {} 3 pkg_1.0.dsc", Hash::Sha256.hex(data)))); + } +} diff --git a/packages/debmagic-common/src/debian/copyright.rs b/packages/debmagic-common/src/debian/copyright.rs new file mode 100644 index 00000000..53a6168b --- /dev/null +++ b/packages/debmagic-common/src/debian/copyright.rs @@ -0,0 +1,99 @@ +/// The `Files-Excluded` patterns from a `debian/copyright` file. +/// +/// The field is a machine-readable header (not part of the deb822 +/// paragraphs): `Files-Excluded:` for the main tree and +/// `Files-Excluded-:` for a multiple-upstream-tarballs +/// component, each with indented or same-line glob patterns. +#[derive(Debug, Clone, Default)] +pub struct FilesExcluded { + /// Patterns of the main `Files-Excluded:` field. + pub main: Vec, + /// Patterns per component, keyed by component name. + pub components: std::collections::HashMap>, +} + +/// Parse the `Files-Excluded` fields from a `debian/copyright` +/// file's content. +pub fn files_excluded(content: &str) -> FilesExcluded { + let mut result = FilesExcluded::default(); + // the component whose patterns we are collecting; `None` = main + let mut current: Option> = None; + + for line in content.lines() { + let trimmed = line.trim(); + if let Some(field) = trimmed.strip_prefix("Files-Excluded") { + // `Files-Excluded:` (main) or `Files-Excluded-:` + let (component, rest) = match field.strip_prefix(':') { + Some(rest) => (None, rest), + None => match field.strip_prefix('-').and_then(|f| f.split_once(':')) { + Some((component, rest)) + if !component.is_empty() && !component.contains(char::is_whitespace) => + { + (Some(component.to_string()), rest) + } + _ => { + current = None; + continue; + } + }, + }; + current = Some(component.clone()); + let target = match &component { + Some(component) => result.components.entry(component.clone()).or_default(), + None => &mut result.main, + }; + let inline = rest.trim(); + if !inline.is_empty() { + target.push(inline.to_string()); + } + continue; + } + if trimmed.is_empty() { + current = None; + continue; + } + if let Some(component) = ¤t { + // patterns are indented under the field + if line.starts_with(' ') || line.starts_with('\t') { + let target = match component { + Some(component) => result.components.entry(component.clone()).or_default(), + None => &mut result.main, + }; + target.push(trimmed.to_string()); + } else { + current = None; + } + } + } + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_files_excluded_main() { + let content = "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\nFiles-Excluded:\n win32/*\n docs/html/*\n\nFiles: *\nCopyright: X\nLicense: MIT\n"; + let excluded = files_excluded(content); + assert_eq!(excluded.main, vec!["win32/*", "docs/html/*"]); + assert!(excluded.components.is_empty()); + } + + #[test] + fn test_files_excluded_inline_and_component() { + let content = + "Files-Excluded: win32/*\nFiles-Excluded-doc:\n doc/*\nFiles: *\nCopyright: X\n"; + let excluded = files_excluded(content); + assert_eq!(excluded.main, vec!["win32/*"]); + assert_eq!(excluded.components["doc"], vec!["doc/*"]); + } + + #[test] + fn test_files_excluded_none() { + let content = "Files: *\nCopyright: X\nLicense: MIT\n"; + let excluded = files_excluded(content); + assert!(excluded.main.is_empty()); + assert!(excluded.components.is_empty()); + } +} diff --git a/packages/debmagic-common/src/debian/source.rs b/packages/debmagic-common/src/debian/source.rs new file mode 100644 index 00000000..0952e02d --- /dev/null +++ b/packages/debmagic-common/src/debian/source.rs @@ -0,0 +1,64 @@ +/// The source package format from `debian/source/format`. +/// +/// dpkg-source defaults to `1.0` when the file is absent. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum SourceFormat { + #[default] + V1, + Quilt, + Native, +} + +impl SourceFormat { + /// Parse the content of a `debian/source/format` file; anything + /// but the known 3.0 formats is treated as `1.0`, like dpkg-source. + pub fn parse(content: &str) -> Self { + match content.trim() { + "3.0 (native)" => Self::Native, + "3.0 (quilt)" => Self::Quilt, + _ => Self::V1, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::V1 => "1.0", + Self::Quilt => "3.0 (quilt)", + Self::Native => "3.0 (native)", + } + } + + /// A native package has no `orig` tarball at all. + pub fn is_native(self) -> bool { + self == Self::Native + } +} + +#[cfg(test)] +mod tests { + use super::*; + use test_case::test_case; + + #[test_case("3.0 (native)\n", SourceFormat::Native; "native")] + #[test_case("3.0 (quilt)", SourceFormat::Quilt; "quilt")] + #[test_case("1.0\n", SourceFormat::V1; "v1")] + #[test_case("garbage", SourceFormat::V1; "unknown is v1")] + #[test_case("", SourceFormat::V1; "empty is v1")] + fn test_parse(content: &str, expected: SourceFormat) { + assert_eq!(SourceFormat::parse(content), expected); + } + + #[test] + fn test_as_str_roundtrip() { + for format in [SourceFormat::V1, SourceFormat::Quilt, SourceFormat::Native] { + assert_eq!(SourceFormat::parse(format.as_str()), format); + } + } + + #[test] + fn test_is_native() { + assert!(SourceFormat::Native.is_native()); + assert!(!SourceFormat::Quilt.is_native()); + assert!(!SourceFormat::V1.is_native()); + } +} diff --git a/packages/debmagic-common/src/debian/version.rs b/packages/debmagic-common/src/debian/version.rs index a6e5400d..a68d4dfa 100644 --- a/packages/debmagic-common/src/debian/version.rs +++ b/packages/debmagic-common/src/debian/version.rs @@ -67,6 +67,97 @@ impl fmt::Display for PackageVersion { } } +impl PartialOrd for PackageVersion { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for PackageVersion { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + let epoch = self.epoch.unwrap_or(0); + let other_epoch = other.epoch.unwrap_or(0); + epoch + .cmp(&other_epoch) + .then_with(|| compare_version_parts(&self.upstream, &other.upstream)) + .then_with(|| { + compare_version_parts( + self.revision.as_deref().unwrap_or(""), + other.revision.as_deref().unwrap_or(""), + ) + }) + } +} + +/// Compare one non-epoch part of a debian version (upstream or revision) +/// with dpkg semantics: alternating non-digit / digit chunks, non-digits +/// compare lexically with `~` sorting before everything including the end. +fn compare_version_parts(a: &str, b: &str) -> std::cmp::Ordering { + let mut a = a; + let mut b = b; + loop { + let a_empty = a.is_empty(); + let b_empty = b.is_empty(); + if a_empty && b_empty { + return std::cmp::Ordering::Equal; + } + // `~` sorts before everything, including the empty remainder + let a_tilde = a.starts_with('~'); + let b_tilde = b.starts_with('~'); + match (a_tilde, b_tilde) { + (true, false) => return std::cmp::Ordering::Less, + (false, true) => return std::cmp::Ordering::Greater, + (true, true) => { + a = &a[1..]; + b = &b[1..]; + continue; + } + (false, false) => {} + } + if a_empty { + return std::cmp::Ordering::Less; + } + if b_empty { + return std::cmp::Ordering::Greater; + } + // non-digit prefix: compare lexically until digits start on either side + let a_nondigits = non_digit_prefix_len(a); + let b_nondigits = non_digit_prefix_len(b); + if a_nondigits > 0 || b_nondigits > 0 { + let a_prefix = &a[..a_nondigits]; + let b_prefix = &b[..b_nondigits]; + match a_prefix.cmp(b_prefix) { + std::cmp::Ordering::Equal => { + a = &a[a_nondigits..]; + b = &b[b_nondigits..]; + } + other => return other, + } + } else { + // both start with digits: compare numerically + let a_digits = digit_prefix_len(a); + let b_digits = digit_prefix_len(b); + let a_num: u64 = a[..a_digits].parse().unwrap_or(u64::MAX); + let b_num: u64 = b[..b_digits].parse().unwrap_or(u64::MAX); + match a_num.cmp(&b_num) { + std::cmp::Ordering::Equal => { + a = &a[a_digits..]; + b = &b[b_digits..]; + } + other => return other, + } + } + } +} + +fn non_digit_prefix_len(s: &str) -> usize { + s.find(|c: char| c.is_ascii_digit()).unwrap_or(s.len()) +} + +fn digit_prefix_len(s: &str) -> usize { + s.find(|c: char| !c.is_ascii_digit()).unwrap_or(s.len()) +} + #[derive(Debug, PartialEq, Eq)] pub struct VersionParseError; @@ -138,4 +229,21 @@ mod tests { // reverse formatting works as well assert_eq!(parsed_version.version(), version); } + + #[test_case("1.0", "1.0", std::cmp::Ordering::Equal; "equal")] + #[test_case("1.1", "1.0", std::cmp::Ordering::Greater; "minor bump")] + #[test_case("2.0", "10.0", std::cmp::Ordering::Less; "numeric not lexical")] + #[test_case("1.0~rc1", "1.0", std::cmp::Ordering::Less; "prerelease before release")] + #[test_case("1.0~rc1", "1.0~rc2", std::cmp::Ordering::Less; "prerelease order")] + #[test_case("1.0-1", "1.0-1", std::cmp::Ordering::Equal; "revision equal")] + #[test_case("1.0-2", "1.0-1", std::cmp::Ordering::Greater; "revision order")] + #[test_case("1.0-1ubuntu1", "1.0-1", std::cmp::Ordering::Greater; "ubuntu after debian")] + #[test_case("1.0+dfsg1", "1.0", std::cmp::Ordering::Greater; "dfsg suffix after")] + #[test_case("1:0.9", "2.0", std::cmp::Ordering::Greater; "epoch wins")] + #[test_case("1.2.3a.4-42.2-14ubuntu2", "1.2.3a.4-42.2-14ubuntu3", std::cmp::Ordering::Less; "complex")] + fn test_version_compare(a: &str, b: &str, expected: std::cmp::Ordering) { + let a = PackageVersion::from_str(a).unwrap(); + let b = PackageVersion::from_str(b).unwrap(); + assert_eq!(a.cmp(&b), expected); + } } diff --git a/packages/debmagic-common/src/lib.rs b/packages/debmagic-common/src/lib.rs index e4de1277..44ece3ad 100644 --- a/packages/debmagic-common/src/lib.rs +++ b/packages/debmagic-common/src/lib.rs @@ -1,2 +1,4 @@ +pub mod changes; pub mod debian; pub mod distro; +pub mod package; diff --git a/packages/debmagic-common/src/package.rs b/packages/debmagic-common/src/package.rs new file mode 100644 index 00000000..d433c247 --- /dev/null +++ b/packages/debmagic-common/src/package.rs @@ -0,0 +1,264 @@ +use std::path::{Path, PathBuf}; +use std::rc::Rc; + +use crate::debian::changelog::ChangelogHead; +use crate::debian::copyright::FilesExcluded; +use crate::debian::source::SourceFormat; +use crate::debian::version::PackageVersion; + +/// Reads the raw content of a `debian/` metadata file, given its +/// location relative to the package root (e.g. `"changelog"`, +/// `"source/format"`, `"copyright"`). Implemented by the consumer, +/// so this crate never touches the filesystem. +pub type FileReader = Box anyhow::Result>; + +/// Where a package's files come from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Location { + /// A source tree on disk; `debian/` lives inside this dir. + SourceDir(PathBuf), + /// In-memory contents, e.g. a test fixture built with + /// [`SourcePackage::from_files`]. + InMemory, +} + +/// A Debian source package, as modeled from its `debian/` metadata. +/// +/// The aggregate of the parsed substructures; the entry point for +/// anything that asks "what is this package". Substructures are read +/// lazily through the [`FileReader`] the consumer supplies at +/// construction, so I/O stays outside this crate and only the +/// metadata actually requested is ever read. +#[derive(Clone)] +pub struct SourcePackage { + reader: Rc, + location: Location, + changelog: ChangelogHead, + source_format: SourceFormat, +} + +impl std::fmt::Debug for SourcePackage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SourcePackage") + .field("location", &self.location) + .field("changelog", &self.changelog) + .field("source_format", &self.source_format) + .finish() + } +} + +impl SourcePackage { + /// Build the model from a reader of the package's `debian/` + /// metadata files and where those files live. Reads `changelog` + /// and `source/format` eagerly (every consumer needs them); + /// everything else stays lazy. + pub fn from_reader(reader: Rc, location: Location) -> anyhow::Result { + let changelog_content = reader("changelog")?; + let changelog = changelog_content + .parse::() + .map_err(|error| anyhow::anyhow!("failed to parse changelog: {error}"))?; + let changelog = ChangelogHead::from_changelog(&changelog).ok_or_else(|| { + anyhow::anyhow!("changelog head entry has no package, version or distribution") + })?; + let source_format = reader("source/format") + .map(|content| SourceFormat::parse(&content)) + .unwrap_or_default(); + Ok(Self { + reader, + location, + changelog, + source_format, + }) + } + + /// Build the model from in-memory file contents, keyed by their + /// `debian/`-relative name (`"changelog"`, `"source/format"`, ...). + /// The file-less construction for tests and other synthetic + /// packages; a file absent from the map behaves like one missing + /// on disk. + pub fn from_files(files: I) -> anyhow::Result + where + I: IntoIterator, + S: Into, + { + let files: std::collections::HashMap = files + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(); + let reader: FileReader = Box::new(move |name: &str| { + files + .get(name) + .cloned() + .ok_or_else(|| anyhow::anyhow!("no such file: {name}")) + }); + SourcePackage::from_reader(Rc::new(reader), Location::InMemory) + } + + /// The source package name. + pub fn name(&self) -> &str { + &self.changelog.package + } + + /// The version of the latest changelog entry. + pub fn version(&self) -> &PackageVersion { + &self.changelog.version + } + + /// The raw distribution names of the latest changelog entry (not resolved + /// to a [`crate::distro::DistroVersion`]). + pub fn distributions(&self) -> &[String] { + &self.changelog.distributions + } + + /// A native package has no `orig` tarball at all. + pub fn is_native(&self) -> bool { + self.source_format.is_native() + } + + /// Where the package's files live; the source tree dir for a + /// package opened from disk. + pub fn location(&self) -> &Location { + &self.location + } + + /// The source tree dir, for a package opened from disk. + pub fn source_dir(&self) -> anyhow::Result<&Path> { + match &self.location { + Location::SourceDir(dir) => Ok(dir), + Location::InMemory => Err(anyhow::anyhow!( + "this package has no source dir; it was built from in-memory files" + )), + } + } + + /// The `Files-Excluded` patterns from `debian/copyright`. + /// Read lazily; a missing file yields no excludes. + pub fn copyright_excludes(&self) -> anyhow::Result { + Ok((self.reader)("copyright") + .map(|content| crate::debian::copyright::files_excluded(&content)) + .unwrap_or_default()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + use std::collections::HashMap; + + /// A reader over an in-memory file map that records which files + /// were read, so tests can assert laziness. + fn reader(files: &[(&str, &str)]) -> (Rc, Rc>>) { + let files: HashMap = files + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + let reads = Rc::new(RefCell::new(Vec::new())); + let reads_clone = reads.clone(); + let reader: FileReader = Box::new(move |name: &str| { + reads_clone.borrow_mut().push(name.to_string()); + files + .get(name) + .cloned() + .ok_or_else(|| anyhow::anyhow!("no such file: {name}")) + }); + (Rc::new(reader), reads) + } + + const CHANGELOG: &str = "postfix (3.11.7-1) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n"; + + #[test] + fn test_accessors() { + let (reader, reads) = + reader(&[("changelog", CHANGELOG), ("source/format", "3.0 (quilt)\n")]); + let package = SourcePackage::from_reader(reader, Location::InMemory).unwrap(); + assert_eq!(package.name(), "postfix"); + assert_eq!(package.version().version(), "3.11.7-1"); + assert_eq!(package.distributions(), ["unstable"]); + assert!(!package.is_native()); + // construction reads only the eager files + assert_eq!(*reads.borrow(), ["changelog", "source/format"]); + } + + #[test] + fn test_native() { + let (reader, _) = reader(&[ + ("changelog", CHANGELOG), + ("source/format", "3.0 (native)\n"), + ]); + assert!( + SourcePackage::from_reader(reader, Location::InMemory) + .unwrap() + .is_native() + ); + } + + #[test] + fn test_copyright_excludes_lazy() { + let (reader, reads) = reader(&[ + ("changelog", CHANGELOG), + ("source/format", "3.0 (quilt)\n"), + ("copyright", "Files-Excluded:\n win32/*\n"), + ]); + let package = SourcePackage::from_reader(reader, Location::InMemory).unwrap(); + assert_eq!(*reads.borrow(), ["changelog", "source/format"]); + let excludes = package.copyright_excludes().unwrap(); + assert_eq!(excludes.main, ["win32/*"]); + assert_eq!(*reads.borrow(), ["changelog", "source/format", "copyright"]); + } + + #[test] + fn test_copyright_excludes_missing_file() { + let (reader, _) = reader(&[("changelog", CHANGELOG), ("source/format", "3.0 (quilt)\n")]); + let package = SourcePackage::from_reader(reader, Location::InMemory).unwrap(); + // a missing copyright is no error, just no excludes + assert!(package.copyright_excludes().unwrap().main.is_empty()); + } + + #[test] + fn test_from_files() { + let package = SourcePackage::from_files([ + ("changelog", CHANGELOG), + ("source/format", "3.0 (quilt)\n"), + ("copyright", "Files-Excluded:\n win32/*\n"), + ]) + .unwrap(); + assert_eq!(package.name(), "postfix"); + assert!(!package.is_native()); + assert_eq!(package.copyright_excludes().unwrap().main, ["win32/*"]); + } + + #[test] + fn test_from_files_missing_lazy_file() { + // a file absent from the map behaves like one missing on disk + let package = SourcePackage::from_files([ + ("changelog", CHANGELOG), + ("source/format", "3.0 (quilt)\n"), + ]) + .unwrap(); + assert!(package.copyright_excludes().unwrap().main.is_empty()); + } + + #[test] + fn test_from_files_missing_changelog() { + assert!(SourcePackage::from_files([("source/format", "3.0 (quilt)\n")]).is_err()); + } + + #[test] + fn test_source_dir() { + let (reader, _) = reader(&[("changelog", CHANGELOG), ("source/format", "3.0 (quilt)\n")]); + let package = + SourcePackage::from_reader(reader, Location::SourceDir(PathBuf::from("/tmp/pkg"))) + .unwrap(); + assert_eq!(package.source_dir().unwrap(), Path::new("/tmp/pkg")); + assert_eq!( + package.location(), + &Location::SourceDir(PathBuf::from("/tmp/pkg")) + ); + + // an in-memory package has no source dir + let package = SourcePackage::from_files([("changelog", CHANGELOG)]).unwrap(); + assert!(package.source_dir().is_err()); + assert_eq!(package.location(), &Location::InMemory); + } +} diff --git a/packages/debmagic-dpkg-driver/Debmagic.pm b/packages/debmagic-dpkg-driver/Debmagic.pm new file mode 100644 index 00000000..67840086 --- /dev/null +++ b/packages/debmagic-dpkg-driver/Debmagic.pm @@ -0,0 +1,126 @@ +# Copyright © 2026 Michael Loipführer +# Copyright © 2026 Jonas Jelten +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +=encoding utf8 + +=head1 NAME + +Dpkg::BuildDriver::Debmagic - build a Debian package using debmagic + +=head1 DESCRIPTION + +This class is used by dpkg-buildpackage to drive the build of a Debian +package whose packaging instructions are written in Python with the +debmagic packaging API, in F. + +It is selected by setting the B field in F +to I: + + Source: mypackage + ... + Build-Driver: debmagic + Build-Depends: debmagic-pkg, debmagic-dpkg-driver + +The package needs to build-depend on I (the python +packaging API imported by F) and on +I (this module), since dpkg-buildpackage loads +the driver from the build environment. + +The driver executes F with the dpkg build tasks +(clean, build, build-arch, build-indep, binary, binary-arch, +binary-indep and custom targets). + +Root handling (B, gain-root-command, fakeroot) is +inherited from L, so debmagic packages +behave like any other package from dpkg-buildpackage's point of view. + +B: This is a private module, its API can change at any time. + +=cut + +package Dpkg::BuildDriver::Debmagic 0.01; + +use v5.36; + +use parent qw(Dpkg::BuildDriver::DebianRules); + +use Dpkg::Gettext; +use Dpkg::ErrorHandling; +use Dpkg::Path qw(find_command); + +=head1 METHODS + +=over 4 + +=item $bd = Dpkg::BuildDriver::Debmagic->new(%opts) + +Create a new Dpkg::BuildDriver::Debmagic object. + +When a package has B, the build recipe will be loaded from F by this driver. +The driver supports the same options as L. +The path to the rules file can be overridden with `dpkg-buildpackage -R B`. + +=cut + +sub new { + my ($this, %opts) = @_; + my $class = ref($this) || $this; + + # dpkg-buildpackage provides its default rules path of 'debian/rules'. + # an explicit -R override can be used to customize the load path. + my $rules = $opts{debian_rules} // [ 'debian/rules.py' ]; + if (@{$rules} == 1 && $rules->[0] eq 'debian/rules') { + $opts{debian_rules} = [ 'debian/rules.py' ]; + } + + return $class->SUPER::new(%opts); +} + +=item $bd->pre_check() + +Perform build driver specific checks, before anything else. + +Apart from the F checks inherited from +L (which apply to F +alike), this verifies that a python3 interpreter is available. + +=cut + +sub pre_check { + my $self = shift; + + $self->SUPER::pre_check(); + + error(g_('this package requires the debmagic build driver, ' . + 'but python3 is not installed')) + unless find_command('python3'); + + return; +} + +=back + +=head1 CHANGES + +=head2 Version 0.01 + +First version, marked private since the Dpkg::BuildDriver interface +itself is still experimental upstream. + +=cut + +1; + diff --git a/packages/debmagic-pkg/CHANGELOG.md b/packages/debmagic-pkg/CHANGELOG.md index 1bea339a..279f3e63 100644 --- a/packages/debmagic-pkg/CHANGELOG.md +++ b/packages/debmagic-pkg/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- the `test` build stage is skipped when `DEB_BUILD_OPTIONS` contains `nocheck` + ## [0.0.1-alpha.8] - 2026-09-20 ## [0.0.1-alpha.7] - 2026-08-25 diff --git a/packages/debmagic-pkg/src/debmagic/v0/_build.py b/packages/debmagic-pkg/src/debmagic/v0/_build.py index aea801ff..a6175fe8 100644 --- a/packages/debmagic-pkg/src/debmagic/v0/_build.py +++ b/packages/debmagic-pkg/src/debmagic/v0/_build.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import shutil import subprocess import typing @@ -77,7 +78,16 @@ def run( ) -> None: internal_stages = InternalPreset() + # dpkg-buildpackage exports DEB_BUILD_OPTIONS; nocheck skips the test + # stage entirely, like debhelper's dh_auto_test does. + skip_tests = "nocheck" in os.environ.get("DEB_BUILD_OPTIONS", "").split() + for stage in BuildStage: + if stage is BuildStage.test and skip_tests: + print("debmagic: stage test: skipped (DEB_BUILD_OPTIONS=nocheck)") + self._mark_stage_done(stage) + continue + print(f"debmagic: stage {stage!s}", end="") # skip done stages diff --git a/packages/debmagic/CHANGELOG.md b/packages/debmagic/CHANGELOG.md index 7e3b63b9..059f7682 100644 --- a/packages/debmagic/CHANGELOG.md +++ b/packages/debmagic/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- `debmagic switch ` upstream source fetching and orig tarballs handling with `debmagic upstream` (and caching them) +- dynamic interval for running `apt update` configured by `--apt-update-age` / `driver.apt_update_age` + ## [0.0.1-alpha.8] - 2026-09-20 - publish the Rust crates to crates.io on tagged releases diff --git a/packages/debmagic/Cargo.toml b/packages/debmagic/Cargo.toml index 1726bc59..f7569c23 100644 --- a/packages/debmagic/Cargo.toml +++ b/packages/debmagic/Cargo.toml @@ -20,9 +20,14 @@ glob = { workspace = true } libc = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +chrono = { workspace = true } toml = { workspace = true } toml_edit = { workspace = true } uuid = { workspace = true, features = ["v4"] } +regex = { workspace = true } +reqwest = { workspace = true } +futures-util = { workspace = true } +tokio = { workspace = true } debian-changelog = { workspace = true } debian-control = { workspace = true } deb822-lossless = { workspace = true } @@ -30,3 +35,6 @@ ignore = { workspace = true } md-5 = { workspace = true } sha1 = { workspace = true } sha2 = { workspace = true } + +[dev-dependencies] +test-case = { workspace = true } diff --git a/packages/debmagic/src/build/artifacts.rs b/packages/debmagic/src/build/artifacts.rs index 8041b08e..b18cd04f 100644 --- a/packages/debmagic/src/build/artifacts.rs +++ b/packages/debmagic/src/build/artifacts.rs @@ -5,7 +5,6 @@ use std::{ }; use anyhow::{Context, anyhow, bail}; -use debian_control::lossless::changes::Changes; /// Locate the single `.changes` file in a build work directory. pub fn find_changes_file(build_dir: &Path) -> anyhow::Result { @@ -63,8 +62,7 @@ pub fn export_build_artifacts(build_dir: &Path, output_dir: &Path) -> anyhow::Re changes_path.display() ); } - let changes = Changes::from_file(&changes_path) - .with_context(|| format!("failed to parse {}", changes_path.display()))?; + let changes = crate::control::read_changes(&changes_path)?; let files = changes .files() .ok_or_else(|| anyhow!("{} has no Files field", changes_path.display()))?; @@ -125,8 +123,7 @@ pub fn copy_changes_artifacts(changes_path: &Path, dest_dir: &Path) -> anyhow::R changes_path.display() ) })?; - let changes = Changes::from_file(changes_path) - .with_context(|| format!("failed to parse {}", changes_path.display()))?; + let changes = crate::control::read_changes(changes_path)?; let files = changes .files() .ok_or_else(|| anyhow!("{} has no Files field", changes_path.display()))?; diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index 95043007..1d3173fd 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -14,11 +14,9 @@ use crate::driver::{ SignRequest, config::DriverConfig, create_driver, create_driver_from_metadata, remove_environment_root, }; -use crate::{ - config::Config, - package::{PackageIdentity, PackageTarget}, -}; +use crate::{config::Config, package::PackageTarget}; use anyhow::{Context, anyhow}; +use debmagic_common::package::SourcePackage; pub mod artifacts; pub mod attach; @@ -34,6 +32,7 @@ struct Build { sign_package: bool, clean: bool, build_debug_symbols: bool, + run_test: bool, host_arch_variant: Option, } @@ -53,6 +52,7 @@ impl Build { sign_package: intent.config.sign.source, clean: intent.config.clean, build_debug_symbols: intent.config.build_debug_symbols, + run_test: intent.config.run_test, host_arch_variant: intent.config.host_arch_variant.clone(), }) } @@ -92,6 +92,7 @@ impl Build { sign_package: false, clean: false, build_debug_symbols: false, + run_test: true, host_arch_variant: None, }) } @@ -119,20 +120,20 @@ impl Build { fn get_build_root_and_identifier( temp_build_dir: &Path, - identity: &PackageIdentity, + package: &SourcePackage, ) -> (String, PathBuf) { - let package_identifier = format!("{}-{}", identity.name, identity.version); + let package_identifier = format!("{}-{}", package.name(), package.version()); let build_root = temp_build_dir.join(&package_identifier); (package_identifier, build_root) } fn prepare_build_env(intent: &BuildIntent, target: &PackageTarget) -> anyhow::Result { let (package_identifier, build_root) = - get_build_root_and_identifier(&intent.config.temp_build_dir, &target.identity); + get_build_root_and_identifier(&intent.config.temp_build_dir, &target.package); let environment = Environment { driver: intent.driver, - package_name: target.identity.name.clone(), + package_name: target.package.name().to_string(), package_identifier, root_dir: build_root.clone(), distro: target.distro.clone(), @@ -165,7 +166,7 @@ fn prepare_build_env(intent: &BuildIntent, target: &PackageTarget) -> anyhow::Re .context("failed to create build directories")?; stage_source_tree( &environment, - &target.identity, + &target.package, intent.config.source_sync_mode, incremental, )?; @@ -182,7 +183,7 @@ fn prepare_build_env(intent: &BuildIntent, target: &PackageTarget) -> anyhow::Re crate::output::step("Staging source tree"); stage_source_tree( &environment, - &target.identity, + &target.package, intent.config.source_sync_mode, incremental, )?; @@ -190,9 +191,9 @@ fn prepare_build_env(intent: &BuildIntent, target: &PackageTarget) -> anyhow::Re Build::create(environment, intent) } -pub fn get_shell_in_build(config: &Config, identity: &PackageIdentity) -> anyhow::Result<()> { +pub fn get_shell_in_build(config: &Config, package: &SourcePackage) -> anyhow::Result<()> { let (_package_identifier, build_root) = - get_build_root_and_identifier(&config.temp_build_dir, identity); + get_build_root_and_identifier(&config.temp_build_dir, package); let build = Build::from_build_root(&build_root, &config.driver)?; let result = build .driver @@ -204,15 +205,18 @@ pub fn get_shell_in_build(config: &Config, identity: &PackageIdentity) -> anyhow Ok(()) } -fn deb_build_options(existing: Option<&str>, build_debug_symbols: bool) -> String { +fn deb_build_options(existing: Option<&str>, build_debug_symbols: bool, run_test: bool) -> String { let mut options = existing .unwrap_or_default() .split_whitespace() - .filter(|option| *option != "noautodbgsym") + .filter(|option| *option != "noautodbgsym" && *option != "nocheck") .collect::>(); if !build_debug_symbols { options.push("noautodbgsym"); } + if !run_test { + options.push("nocheck"); + } options.join(" ") } @@ -234,10 +238,11 @@ fn run_build( ) -> anyhow::Result<()> { let sign = &request.intent.config.sign; - let package = &request.target.identity; + let package = &request.target.package; crate::output::stage(&format!( "Preparing build environment for {} {}", - package.name, package.version + package.name(), + package.version() )); let build = prepare_build_env(request.intent, request.target) .context("failed to prepare build environment")?; @@ -267,7 +272,7 @@ fn run_build( changes_file: &changes_file, sign_key: sign.key.as_deref(), sign_tool: sign.tool, - sign_command: sign.command.as_deref(), + sign_command: sign.sign_command.as_deref(), notify: sign.notify, package: &build.environment.package_identifier, })?; @@ -308,7 +313,11 @@ fn run_build( Ok(()) } -pub fn build_package(intent: &BuildIntent, target: &PackageTarget) -> anyhow::Result<()> { +pub fn build_package( + intent: &BuildIntent, + target: &PackageTarget, + changes_options: &[String], +) -> anyhow::Result<()> { let request = BuildRequest { intent, target }; run_build(&request, |build| { crate::output::stage("Building binary packages"); @@ -328,18 +337,32 @@ pub fn build_package(intent: &BuildIntent, target: &PackageTarget) -> anyhow::Re &[], )?; let inherited_options = std::env::var("DEB_BUILD_OPTIONS").ok(); - let options = deb_build_options(inherited_options.as_deref(), build.build_debug_symbols); + let options = deb_build_options( + inherited_options.as_deref(), + build.build_debug_symbols, + build.run_test, + ); let mut env_add = vec![("DEB_BUILD_OPTIONS", options.as_str())]; if let Some(variant) = build.host_arch_variant.as_deref() { env_add.push(("DEB_HOST_ARCH_VARIANT", variant)); } - let mut dpkg_buildpackage_args = vec!["dpkg-buildpackage", "-us", "-uc", "-ui"]; + let mut dpkg_buildpackage_args: Vec = vec![ + "dpkg-buildpackage".into(), + "-us".into(), + "-uc".into(), + "-ui".into(), + ]; if !build.clean { // Non-incremental builds already stage a clean source tree, while // incremental builds preserve their outputs intentionally. - dpkg_buildpackage_args.push("-nc"); + dpkg_buildpackage_args.push("-nc".into()); + } + for option in changes_options { + dpkg_buildpackage_args.push(format!("--changes-option={option}")); } - dpkg_buildpackage_args.push("-b"); + dpkg_buildpackage_args.push("-b".into()); + let dpkg_buildpackage_args: Vec<&str> = + dpkg_buildpackage_args.iter().map(String::as_str).collect(); build.driver.run_command_checked( &dpkg_buildpackage_args, &build.environment.staged_source_dir(), @@ -374,19 +397,80 @@ fn check_dpkg_buildpackage_available() -> anyhow::Result<()> { ) } +/// Make `source` available at `destination` without copying bytes when +/// avoidable: hardlink (same filesystem), then symlink, then copy as the +/// last resort. dpkg-source only reads the file, so a link is fine. +fn stage_file(source: &Path, destination: &Path) -> anyhow::Result<()> { + if destination.exists() { + std::fs::remove_file(destination) + .with_context(|| format!("failed to remove {}", destination.display()))?; + } + if std::fs::hard_link(source, destination).is_ok() { + return Ok(()); + } + if std::os::unix::fs::symlink(source, destination).is_ok() { + return Ok(()); + } + fs::copy(source, destination).map(|_| ()).with_context(|| { + format!( + "failed to stage {} into the build environment", + source.display() + ) + }) +} + /// Build a `.dsc` + tarball + `.buildinfo` + `.changes` source package. /// /// If `config.clean` is set, build-dependencies are installed before /// `dpkg-buildpackage` runs `debian/rules clean` once. -pub fn build_source_package(intent: &BuildIntent, target: &PackageTarget) -> anyhow::Result<()> { +/// `changes_options` are extra `--changes-option=...` arguments passed +/// to `dpkg-buildpackage` verbatim, e.g. +/// `--changes-option=-DVcs-Git=https://...` for git-ubuntu's +/// upload/git correlation. +pub async fn build_source_package( + intent: &BuildIntent, + target: &PackageTarget, + changes_options: &[String], + include_orig: bool, +) -> anyhow::Result<()> { if intent.driver == DriverType::Bare { check_dpkg_buildpackage_available()?; } + // dpkg-source looks for the orig tarball in the parent of the source + // dir it builds, which inside the environment is the work dir. + let orig_fetch_dir = intent + .config + .temp_build_dir + .join("orig") + .join(target.package.name()); + let orig_tarball = crate::upstream::orig::fetch_orig_tarball( + &intent.config.orig_tarball, + &target.package, + &orig_fetch_dir, + ) + .await + .with_context(|| { + format!( + "fetching the orig tarball for {} {} failed", + target.package.name(), + target.package.version().upstream_version() + ) + })?; + let request = BuildRequest { intent, target }; run_build(&request, |build| { crate::output::stage("Building source package"); let staged_source_dir = build.environment.staged_source_dir(); + if let Some(tarball) = &orig_tarball { + // the work dir is the staged source dir's parent, which is where + // dpkg-source looks for the tarball + let destination = build + .environment + .work_dir() + .join(tarball.file_name().expect("orig tarball has a file name")); + stage_file(tarball, &destination)?; + } if build.clean { build.driver.run_command_checked( &["apt-get", "-y", "build-dep", "."], @@ -395,10 +479,27 @@ pub fn build_source_package(intent: &BuildIntent, target: &PackageTarget) -> any &[], )?; } - let mut args = vec!["dpkg-buildpackage", "-S", "-d", "-us", "-uc", "-ui"]; + let mut args: Vec = vec![ + "dpkg-buildpackage".into(), + "-S".into(), + "-d".into(), + "-us".into(), + "-uc".into(), + "-ui".into(), + ]; if !build.clean { - args.push("-nc"); + args.push("-nc".into()); } + // -sa/-sd decide whether the .changes references the orig tarball + args.push(if include_orig { + "-sa".into() + } else { + "-sd".into() + }); + for option in changes_options { + args.push(format!("--changes-option={option}")); + } + let args: Vec<&str> = args.iter().map(String::as_str).collect(); build .driver .run_command_checked(&args, &staged_source_dir, false, &[])?; @@ -411,15 +512,53 @@ pub fn build_source_package(intent: &BuildIntent, target: &PackageTarget) -> any mod tests { use super::*; + #[test] + fn stage_file_prefers_hardlink() { + let dir = std::env::temp_dir().join(format!("debmagic-stage-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let source = dir.join("src.tar"); + std::fs::write(&source, "data").unwrap(); + let destination = dir.join("dest.tar"); + stage_file(&source, &destination).unwrap(); + // same filesystem: a hardlink shares the inode + assert_eq!( + std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&source).unwrap()), + std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&destination).unwrap()) + ); + // restaging replaces the destination + stage_file(&source, &destination).unwrap(); + assert_eq!( + std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&source).unwrap()), + std::os::unix::fs::MetadataExt::ino(&std::fs::metadata(&destination).unwrap()) + ); + std::fs::remove_dir_all(&dir).unwrap(); + } + #[test] fn debug_symbol_option_preserves_other_build_options() { assert_eq!( - deb_build_options(Some("nocheck parallel=8"), false), - "nocheck parallel=8 noautodbgsym" + deb_build_options(Some("nocheck parallel=8"), false, true), + "parallel=8 noautodbgsym" + ); + assert_eq!( + deb_build_options(Some("nocheck noautodbgsym parallel=8"), true, true), + "parallel=8" + ); + } + + #[test] + fn run_test_option_adds_nocheck() { + assert_eq!( + deb_build_options(None, false, false), + "noautodbgsym nocheck" + ); + assert_eq!( + deb_build_options(Some("parallel=8"), true, false), + "parallel=8 nocheck" ); assert_eq!( - deb_build_options(Some("nocheck noautodbgsym parallel=8"), true), - "nocheck parallel=8" + deb_build_options(Some("nocheck parallel=8"), true, true), + "parallel=8" ); } } diff --git a/packages/debmagic/src/build/source.rs b/packages/debmagic/src/build/source.rs index ec1a611c..3827d80d 100644 --- a/packages/debmagic/src/build/source.rs +++ b/packages/debmagic/src/build/source.rs @@ -19,7 +19,7 @@ use glob::glob; use clap::ValueEnum; use crate::driver::Environment; -use crate::package::PackageIdentity; +use debmagic_common::package::SourcePackage; /// Selects which files from the source directory are staged into the build tree. #[derive( @@ -218,6 +218,16 @@ fn tracked_entries(src: &Path, paths: &[PathBuf]) -> anyhow::Result anyhow::Result<()> { - let source_dir = &identity.source_dir; + let source_dir = package.source_dir()?; if source_sync_mode == SourceSyncMode::Tracked { let untracked = git_untracked_paths(source_dir); if !untracked.is_empty() { @@ -450,7 +460,7 @@ pub fn stage_source_tree( if untracked.len() > 20 { eprintln!(" ... and {} more", untracked.len() - 20); } - eprintln!(" git add them or use --source-sync worktree to include them"); + eprintln!(" `git add` them or use `--source-sync worktree` to include them"); } } if incremental && source_manifest_path(environment).is_file() { @@ -466,17 +476,22 @@ pub fn stage_source_tree( let source_parent = source_dir .parent() .ok_or_else(|| anyhow!("source directory has no parent"))?; - let prefix = format!("{}_{}", identity.name, identity.version.upstream_version()); - copy_glob( - source_parent, - &format!("{prefix}.orig.tar.*"), - &environment.work_dir(), - )?; - copy_glob( - source_parent, - &format!("{prefix}.orig-*.tar.*"), - &environment.work_dir(), - )?; + // the naming knowledge lives in debmagic-common; only the glob + // wildcard for "any compression" is added here + let orig_pattern = format!( + "{}*", + debmagic_common::changes::orig_prefix(package.name(), package.version().upstream_version()) + ); + let component_pattern = format!( + "{}*", + debmagic_common::changes::component_orig_prefix( + package.name(), + package.version().upstream_version(), + "*" + ) + ); + copy_glob(source_parent, &orig_pattern, &environment.work_dir())?; + copy_glob(source_parent, &component_pattern, &environment.work_dir())?; Ok(()) } @@ -641,6 +656,20 @@ mod tests { Ok(()) } + #[test] + fn tracked_sync_skips_files_deleted_from_worktree() -> anyhow::Result<()> { + let repo = git_test_repo()?; + fs::remove_file(repo.join("debian/control"))?; + + let entries = source_tree_entries(&repo, SourceSyncMode::Tracked)?; + let paths: Vec<&Path> = entries.iter().map(|e| e.path.as_path()).collect(); + assert!(!paths.contains(&Path::new("debian/control"))); + assert!(paths.contains(&Path::new("debian"))); + + fs::remove_dir_all(repo)?; + Ok(()) + } + #[test] fn tracked_sync_falls_back_outside_git_worktree() -> anyhow::Result<()> { let dir = std::env::temp_dir().join(format!("debmagic-nogit-{}", uuid::Uuid::new_v4())); diff --git a/packages/debmagic/src/build_intent.rs b/packages/debmagic/src/build_intent.rs index 5b2d9d86..4b490199 100644 --- a/packages/debmagic/src/build_intent.rs +++ b/packages/debmagic/src/build_intent.rs @@ -21,6 +21,7 @@ pub struct BuildIntentInput { pub persistent: Option, pub incremental: Option, pub debug_symbols: Option, + pub test: Option, pub sign: Option, pub sign_key: Option, pub sign_tool: Option, @@ -70,6 +71,9 @@ pub fn resolve_build_intent(input: BuildIntentInput) -> anyhow::Result anyhow::Result SourceFormat { + std::fs::read_to_string(dir.join("debian").join("source").join("format")) + .map(|content| SourceFormat::parse(&content)) + .unwrap_or_default() +} + +/// Read and parse `debian/changelog`. +pub fn load_changelog(dir: &Path) -> anyhow::Result { + let path = dir.join("debian").join("changelog"); + let content = std::fs::read_to_string(&path) + .with_context(|| format!("failed to read {}", path.display()))?; + content + .parse() + .with_context(|| format!("failed to parse {}", path.display())) +} + +/// The head entry of `debian/changelog`, as the package identity. +pub fn load_changelog_head( + dir: &Path, +) -> anyhow::Result { + let changelog = load_changelog(dir)?; + changelog_head(&changelog).with_context(|| { + format!( + "failed to read the head entry of {}", + dir.join("debian").join("changelog").display() + ) + }) +} + +/// Extract the head entry of a parsed changelog. +pub fn changelog_head( + changelog: &debian_changelog::ChangeLog, +) -> anyhow::Result { + use anyhow::bail; + + if changelog.iter().next().is_none() { + bail!("changelog is empty"); + } + debmagic_common::debian::changelog::ChangelogHead::from_changelog(changelog).ok_or_else(|| { + anyhow::anyhow!("changelog head entry has no package, version or distribution") + }) +} diff --git a/packages/debmagic/src/changes.rs b/packages/debmagic/src/changes.rs new file mode 100644 index 00000000..8d63ad76 --- /dev/null +++ b/packages/debmagic/src/changes.rs @@ -0,0 +1,76 @@ +use std::path::Path; + +use anyhow::Context; + +use crate::control::{read_control, write_control}; + +/// Add one `orig` tarball entry (or, with `None`, remove all orig +/// entries) from a `.changes` file's `Files:`/`Checksums-*:` listings. +/// Adding requires the tarball to exist next to the `.changes`. +/// Returns whether the file was modified. +pub fn set_changes_orig(changes_file: &Path, orig_name: Option<&str>) -> anyhow::Result { + let dir = changes_file + .parent() + .context("changes file has no parent directory")?; + let mut control = read_control(changes_file)?; + + let tarball_data: Vec = match orig_name { + Some(name) => { + let path = dir.join(name); + std::fs::read(&path) + .with_context(|| format!("failed to read the orig tarball {}", path.display()))? + } + None => Vec::new(), + }; + + let modified = + debmagic_common::changes::set_changes_orig_entries(&mut control, orig_name, &tarball_data); + + if modified { + write_control(&control, changes_file)?; + } + Ok(modified) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn set_changes_orig_adds_and_removes() { + let dir = + std::env::temp_dir().join(format!("debmagic-orig-changes-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let changes = dir.join("pkg_1.0-1_source.changes"); + std::fs::write( + &changes, + "Format: 1.8\nSource: pkg\nFiles:\n abc 3 pkg_1.0-1.debian.tar.xz\n def 4 pkg_1.0-1.dsc\nChecksums-Sha256:\n xyz 3 pkg_1.0-1.debian.tar.xz\n zzz 4 pkg_1.0-1.dsc\n", + ) + .unwrap(); + std::fs::write(dir.join("pkg_1.0.orig.tar.xz"), "tarball").unwrap(); + + // add + assert!(set_changes_orig(&changes, Some("pkg_1.0.orig.tar.xz")).unwrap()); + let content = std::fs::read_to_string(&changes).unwrap(); + assert!(content.contains("pkg_1.0.orig.tar.xz")); + assert_eq!(content.matches("pkg_1.0.orig.tar.xz").count(), 2); + + // adding again is a no-op + assert!(!set_changes_orig(&changes, Some("pkg_1.0.orig.tar.xz")).unwrap()); + + // adding a component orig keeps the main one + std::fs::write(dir.join("pkg_1.0.orig-bar.tar.xz"), "component").unwrap(); + assert!(set_changes_orig(&changes, Some("pkg_1.0.orig-bar.tar.xz")).unwrap()); + let content = std::fs::read_to_string(&changes).unwrap(); + assert!(content.contains("pkg_1.0.orig.tar.xz")); + assert!(content.contains("pkg_1.0.orig-bar.tar.xz")); + + // remove drops all origs, main and component + assert!(set_changes_orig(&changes, None).unwrap()); + let content = std::fs::read_to_string(&changes).unwrap(); + assert!(!content.contains("orig.tar")); + assert!(!content.contains("orig-bar")); + + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 4d0273d0..25d2e4f7 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -3,6 +3,9 @@ use std::path::PathBuf; use crate::build::source::SourceSyncMode; use crate::driver::DriverType; use crate::sign::SignTool; +use crate::time::RefreshPolicy; +use crate::upload::UploadMethod; +use crate::upload::orig::IncludeOrig; use clap::{Args, Parser, Subcommand}; /// When to use colored output. Mirrors common CLI conventions; `auto` is the @@ -49,12 +52,81 @@ pub enum Commands { Check(CheckSubcommandArgs), #[command(about = "GPG-sign a .changes file (and its .dsc/.buildinfo) on the host")] Sign(SignSubcommandArgs), + #[command( + about = "Upload a .changes file (and everything it references) to an upload target, dput-style" + )] + Upload(UploadSubcommandArgs), #[command(about = "Inspect the debmagic configuration")] Config(ConfigSubcommandArgs), + #[command(about = "Query and switch upstream versions")] + Upstream(UpstreamSubcommandArgs), #[command(about = "Show version information")] Version {}, } +#[derive(Args, Debug)] +pub struct UpstreamSubcommandArgs { + #[command(subcommand)] + pub command: UpstreamCommands, +} + +#[derive(Subcommand, Debug)] +pub enum UpstreamCommands { + #[command( + about = "List available upstream versions from debian/watch, newest eligible by default" + )] + List(UpstreamListArgs), + #[command( + about = "Switch the package tree to an upstream version: fetch, repack, replace the tree (keeping debian/)" + )] + Switch(UpstreamSwitchArgs), +} + +#[derive(Args, Debug)] +pub struct UpstreamListArgs { + #[arg( + long, + help = "Show all candidate versions, not just the newest newer than the changelog's" + )] + pub all: bool, + + #[arg( + long, + help = "Show the N versions newer than the changelog's, not just the newest" + )] + pub previous: Option, + + #[command(flatten)] + pub common: CommonCli, +} + +#[derive(Args, Debug)] +pub struct UpstreamSwitchArgs { + #[arg(help = "The upstream version to switch to, or 'latest' for the newest eligible")] + pub version: String, + + #[arg( + long, + help = "Only report what would happen, without touching anything" + )] + pub dry_run: bool, + + #[arg( + long, + help = "Skip verifying the upstream tarball signature against debian/upstream/signing-key.asc" + )] + pub no_signature_check: bool, + + #[arg( + long = "verify-command", + help = "Custom verification command for sign.tool = 'custom', run without a shell. Supports {file}, {signature} and {keyring} placeholders; without {signature} the signature path is appended. Defaults to the 'sign.verify_command' setting, falling back to 'sign.sign_command'." + )] + pub verify_command: Option, + + #[command(flatten)] + pub common: CommonCli, +} + #[derive(Args, Debug)] pub struct ConfigSubcommandArgs { #[command(subcommand)] @@ -200,11 +272,24 @@ pub struct CommonBuildArgs { )] pub proposed: Option, + #[arg( + long = "apt-update-age", + help = "When a persistent build environment runs 'apt-get update' again: 'now' (every build), 'never' (only on first creation), or a maximum age of the apt index like '1d' (the default), '12h', '30m'. Fresh environments always update once. Defaults to the 'apt_update_age' setting in the config file. Ignored by the bare driver." + )] + pub apt_update_age: Option, + #[arg( long, - help = "Select the target distribution version, only required if the debian changelog specifies multiple versions" + help = "Target distribution to build for, overriding the changelog's (e.g. 'trixie', 'noble', or a suite declared in base_images). If not provided, use the single distro from changelog." )] pub distro: Option, + + #[arg( + long = "bare-ignore-release", + help = "With the bare driver, build even though the target distro differs from the host's os-release. The host must still provide the build dependencies itself." + )] + pub bare_ignore_release: bool, + #[arg( long = "host-arch-variant", help = "Build for a dpkg architecture variant (e.g. 'amd64v3' on Ubuntu), like dpkg-buildpackage's --host-arch-variant. Sets DEB_HOST_ARCH_VARIANT for the build, which makes the Ubuntu vendor hook append the variant's -march= flags and names the .changes file after the variant. Defaults to the 'host_arch_variant' setting in the config file." @@ -239,7 +324,7 @@ pub struct CommonBuildArgs { #[arg( long = "sign-command", - help = "Custom signing command for --sign-tool custom, run without a shell. Supports {file}, {key} and {email} placeholders; writes the clearsigned result to stdout. Defaults to the 'sign.command' setting in the config file." + help = "Custom signing command for --sign-tool custom, run without a shell. Supports {file}, {key} and {email} placeholders; writes the clearsigned result to stdout. Defaults to the 'sign.sign_command' setting in the config file." )] pub sign_command: Option, @@ -273,6 +358,18 @@ pub struct CommonBuildArgs { #[arg(short, long, help = "Output directory for the package artifacts")] pub output_dir: Option, + + #[arg( + long = "changes-option", + help = "Extra field for the .changes file, passed to dpkg-buildpackage as-is, e.g. --changes-option=-DVcs-Git=https://... (repeatable)" + )] + pub changes_options: Vec, + + #[arg( + long, + help = "After building (and signing, if enabled), upload the resulting .changes to this upload target ('name' or 'name:parameter', e.g. 'ppa:user/repo')" + )] + pub upload: Option, } #[derive(Args, Debug)] @@ -304,12 +401,29 @@ pub struct BinaryTargetArgs { help = "Also build the automatic '-dbgsym' debug symbol package" )] pub debug_symbols: Option, + + #[arg( + long = "test", + num_args = 0..=1, + default_missing_value = "true", + value_parser = clap::value_parser!(bool), + help = "Run the package's test suite during the build. Defaults to the 'run_test' setting in the config file (true if unset); --test=false exports DEB_BUILD_OPTIONS=nocheck so dpkg-buildpackage skips tests." + )] + pub test: Option, } #[derive(Args, Debug)] pub struct SourceTargetArgs { #[command(flatten)] pub build: CommonBuildArgs, + + #[arg( + long = "include-orig", + value_enum, + default_value_t = IncludeOrig::Auto, + help = "Include the orig tarball in the source upload: 'auto' (default) includes it only when the archive cannot have it yet (a new upstream version or a deltarebase onto Debian), 'yes' always, 'no' never" + )] + pub include_orig: IncludeOrig, } #[derive(Args, Debug)] @@ -349,6 +463,12 @@ pub struct TestSubcommandArgs { )] pub proposed: Option, + #[arg( + long = "apt-update-age", + help = "When a persistent test environment runs 'apt-get update' again: 'now' (every run), 'never' (only on first creation), or a maximum age of the apt index like '1d' (the default), '12h', '30m'. Fresh environments always update once. Defaults to the 'apt_update_age' setting in the config file. Ignored by the bare driver." + )] + pub apt_update_age: Option, + #[arg( long, help = "Override the target distribution for the test environment. Defaults to the distro recorded in the prior build's environment.json, not the changelog." @@ -409,7 +529,7 @@ pub struct SignSubcommandArgs { #[arg( long = "sign-command", - help = "Custom signing command for --sign-tool custom, run without a shell. Supports {file}, {key} and {email} placeholders; writes the clearsigned result to stdout. Defaults to the 'sign.command' setting in the config file." + help = "Custom signing command for --sign-tool custom, run without a shell. Supports {file}, {key} and {email} placeholders; writes the clearsigned result to stdout. Defaults to the 'sign.sign_command' setting in the config file." )] pub sign_command: Option, @@ -437,3 +557,65 @@ pub struct SignSubcommandArgs { )] pub file: Option, } + +#[derive(Args, Debug)] +pub struct UploadSubcommandArgs { + #[arg( + help = "Upload target: 'name' or 'name:parameter' (e.g. 'ppa:user/repo'), resolved from [upload.targets] in the config, merging over the builtins (ppa, ubuntu, debian)" + )] + pub target: String, + + #[arg( + long, + value_enum, + help = "Upload method: 'scp' or 'sftp'. Overrides the target's 'method'" + )] + pub method: Option, + + #[arg(long, help = "Server to upload to. Overrides the target's 'server'")] + pub server: Option, + + #[arg( + long, + help = "Remote directory to upload into. Overrides the target's 'incoming'" + )] + pub incoming: Option, + + #[arg( + long, + help = "Login on the remote server. Overrides the target's 'login'" + )] + pub login: Option, + + #[arg(long, help = "Remote port. Overrides the target's 'port'")] + pub port: Option, + + #[arg( + long, + action = clap::ArgAction::SetTrue, + help = "Skip the target's pre_upload_commands" + )] + pub no_hooks: bool, + + #[arg( + long, + action = clap::ArgAction::SetTrue, + help = "Upload even if a successful upload to this target is already recorded" + )] + pub force: bool, + + #[arg( + long = "include-orig", + value_enum, + help = "Include the orig tarball in the upload: 'auto' includes it only when the archive cannot have it yet (a new upstream version or a deltarebase onto Debian), 'yes' always, 'no' never. Rewrites and re-signs the .changes when it disagrees" + )] + pub include_orig: Option, + + #[command(flatten)] + pub common: CommonCli, + + #[arg( + help = "The .changes file to upload; when omitted, located via debian/changelog and the output dir" + )] + pub changes: Option, +} diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 73d538b0..d052b27e 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -3,6 +3,8 @@ use std::path::{Path, PathBuf}; use crate::build::source::SourceSyncMode; use crate::driver::config::DriverConfig; use crate::sign::SignTool; +use crate::upload::UploadConfig; +use crate::upstream::orig::OrigTarballConfig; use anyhow::{Context, anyhow}; use config::{Config as ConfigBuilder, File}; use serde::{Deserialize, Serialize}; @@ -156,8 +158,17 @@ pub struct Config { pub source_sync_mode: SourceSyncMode, /// Always build the automatic `-dbgsym` debug symbol package. pub build_debug_symbols: bool, + /// Run the package's test suite during the build. When false, exports + /// `DEB_BUILD_OPTIONS=nocheck` so dpkg-buildpackage skips tests. + pub run_test: bool, /// Signing of the resulting `.changes`/`.dsc`. pub sign: SignConfig, + /// Named upload targets for `debmagic upload`. + pub upload: UploadConfig, + /// How to fetch the `orig` tarball for source builds. + pub orig_tarball: OrigTarballConfig, + /// `upstream` command behavior. + pub upstream: UpstreamConfig, /// Run `debian/rules clean` before building (like `dpkg-buildpackage` /// does unless passed `-nc`). Disabled by default because non-incremental /// builds already stage a clean source tree and incremental builds preserve @@ -171,6 +182,23 @@ pub struct Config { pub host_arch_variant: Option, } +/// `[upstream]` section: `upstream` command behavior. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default)] +pub struct UpstreamConfig { + /// Verify upstream tarball signatures against + /// `debian/upstream/signing-key.asc` when it exists. + pub verify_signatures: bool, +} + +impl Default for UpstreamConfig { + fn default() -> Self { + Self { + verify_signatures: true, + } + } +} + /// `[sign]` section: whether and how to sign the build artifacts. #[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default)] @@ -183,7 +211,10 @@ pub struct SignConfig { /// Which OpenPGP implementation to use. pub tool: SignTool, /// Custom signing command when `tool` is `custom`, like debsign's `-p`. - pub command: Option, + pub sign_command: Option, + /// Custom verification command when `tool` is `custom`, used for + /// upstream tarball signature checks; falls back to `sign_command`. + pub verify_command: Option, /// Send a desktop notification via `notify-send` just before signing, /// so a hardware-key touch prompt isn't missed. pub notify: bool, @@ -198,7 +229,11 @@ impl Default for Config { incremental: false, source_sync_mode: SourceSyncMode::default(), build_debug_symbols: false, + run_test: true, sign: SignConfig::default(), + upload: UploadConfig::default(), + orig_tarball: OrigTarballConfig::default(), + upstream: UpstreamConfig::default(), clean: false, shell_on_failure: false, host_arch_variant: None, @@ -330,7 +365,7 @@ mod tests { let file = dir.join("sign.toml"); std::fs::write( &file, - "[sign]\nsource = true\nkey = \"you@example.com\"\ncommand = \"gpg --foo\"\nnotify = true\n", + "[sign]\nsource = true\nkey = \"you@example.com\"\nsign_command = \"gpg --foo\"\nnotify = true\n", )?; let cfg = Config::new(&[ConfigPath::new( ConfigLayer::Explicit, @@ -340,7 +375,7 @@ mod tests { std::fs::remove_dir_all(&dir).ok(); assert!(cfg.sign.source); assert_eq!(cfg.sign.key.as_deref(), Some("you@example.com")); - assert_eq!(cfg.sign.command.as_deref(), Some("gpg --foo")); + assert_eq!(cfg.sign.sign_command.as_deref(), Some("gpg --foo")); assert!(cfg.sign.notify); Ok(()) } diff --git a/packages/debmagic/src/control.rs b/packages/debmagic/src/control.rs new file mode 100644 index 00000000..4c510624 --- /dev/null +++ b/packages/debmagic/src/control.rs @@ -0,0 +1,84 @@ +use std::path::Path; + +use anyhow::Context; +use deb822_lossless::Paragraph; +use debian_control::lossless::changes::Changes; +use debian_control::pgp; +use sha2::Digest; + +pub use debmagic_common::debian::control::{ + CHECKSUM_FIELDS, Digests, Hash, child_filename, fixup_checksums, verify_checksums, +}; + +/// Digest `path` streaming, with every algorithm a control file's +/// checksum fields may use. +pub(crate) fn digest_file(path: &Path) -> anyhow::Result { + let mut sha256 = sha2::Sha256::new(); + let mut sha1 = sha1::Sha1::new(); + let mut md5 = md5::Md5::new(); + let mut file = + std::fs::File::open(path).with_context(|| format!("failed to open {}", path.display()))?; + let mut buffer = [0u8; 64 * 1024]; + loop { + let read = std::io::Read::read(&mut file, &mut buffer) + .with_context(|| format!("failed to read {}", path.display()))?; + if read == 0 { + break; + } + let chunk = &buffer[..read]; + sha256.update(chunk); + sha1.update(chunk); + md5.update(chunk); + } + let to_hex = |digest: &[u8]| { + digest + .iter() + .map(|b| format!("{b:02x}")) + .collect::() + }; + Ok(Digests { + md5: to_hex(&md5.finalize()), + sha1: to_hex(&sha1.finalize()), + sha256: to_hex(&sha256.finalize()), + }) +} + +/// Read the single deb822 paragraph of a `.changes`/`.dsc`/`.buildinfo` +/// control file, losslessly, so checksum rewrites preserve the original +/// formatting of untouched fields byte-for-byte. PGP-clearsigned files +/// (like archive `.dsc` files) have their armor stripped first. +pub(crate) fn read_control(path: &Path) -> anyhow::Result { + let content = read_control_content(path)?; + let deb822 = content + .parse::() + .with_context(|| format!("failed to parse {}", path.display()))?; + deb822 + .paragraphs() + .next() + .with_context(|| format!("{} contains no paragraph", path.display())) +} + +/// Read a `.changes` file, losslessly, with PGP armor stripped first — +/// signed `.changes` files are the norm, not the exception. +pub(crate) fn read_changes(path: &Path) -> anyhow::Result { + let content = read_control_content(path)?; + Changes::read(content.as_bytes()).with_context(|| format!("failed to parse {}", path.display())) +} + +/// The file content with the PGP clearsign armor removed when present. +fn read_control_content(path: &Path) -> anyhow::Result { + let content = std::fs::read_to_string(path) + .with_context(|| format!("failed to read {}", path.display()))?; + if content.starts_with("-----BEGIN PGP SIGNED MESSAGE-----") { + let (payload, _) = pgp::strip_pgp_signature(&content) + .with_context(|| format!("failed to strip the signature from {}", path.display()))?; + Ok(payload) + } else { + Ok(content) + } +} + +pub(crate) fn write_control(paragraph: &Paragraph, path: &Path) -> anyhow::Result<()> { + std::fs::write(path, paragraph.to_string()) + .with_context(|| format!("failed to write {}", path.display())) +} diff --git a/packages/debmagic/src/driver/config.rs b/packages/debmagic/src/driver/config.rs index 0ed85b90..a8261a30 100644 --- a/packages/debmagic/src/driver/config.rs +++ b/packages/debmagic/src/driver/config.rs @@ -4,6 +4,7 @@ use crate::driver::DriverType; use crate::driver::driver_bare::{DriverBareConfig, DriverBareConfigOverrides}; use crate::driver::driver_docker::{DriverDockerConfig, DriverDockerConfigOverrides}; use crate::driver::driver_lxd::{DriverLxdConfig, DriverLxdConfigOverrides}; +use crate::time::RefreshPolicy; #[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default)] @@ -18,6 +19,10 @@ pub struct DriverConfig { /// Also enable the `-proposed` pocket. Not used by the bare /// driver, which builds on the host's own sources. pub proposed: bool, + /// How old the apt index in a persistent environment may get before + /// `apt-get update` runs again; fresh environments always update once. + /// Not used by the bare driver, which builds on the host's own sources. + pub apt_update_age: RefreshPolicy, pub docker: DriverDockerConfig, pub bare: DriverBareConfig, pub lxd: DriverLxdConfig, @@ -27,6 +32,7 @@ pub struct DriverConfig { pub struct DriverOverrides { pub apt_mirror: Option, pub proposed: Option, + pub apt_update_age: Option, pub docker: DriverDockerConfigOverrides, pub bare: DriverBareConfigOverrides, pub lxd: DriverLxdConfigOverrides, diff --git a/packages/debmagic/src/driver/driver_bare.rs b/packages/debmagic/src/driver/driver_bare.rs index 83aeda8c..f41ba3e3 100644 --- a/packages/debmagic/src/driver/driver_bare.rs +++ b/packages/debmagic/src/driver/driver_bare.rs @@ -6,6 +6,7 @@ use crate::driver::{ DriverType, Environment, EnvironmentDriver, EnvironmentMetadata, IsolationCapability, SignRequest, config::DriverConfig, }; +use crate::subprocess::{self, Capture, CommandResult}; #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(default)] @@ -54,7 +55,8 @@ impl EnvironmentDriver for DriverBare { cwd: &Path, requires_root: bool, env_add: &[(&str, &str)], - ) -> std::io::Result { + capture: Capture, + ) -> std::io::Result { let mut full_cmd: Vec = Vec::new(); let is_root = unsafe { libc::geteuid() == 0 }; @@ -70,8 +72,7 @@ impl EnvironmentDriver for DriverBare { command.current_dir(cwd); command.envs(env_add.iter().copied()); - let status = command.status()?; - Ok(status.code().unwrap_or(-1)) + subprocess::command(command).capture(capture).run() } fn cleanup(&self) -> anyhow::Result<()> { diff --git a/packages/debmagic/src/driver/driver_docker.rs b/packages/debmagic/src/driver/driver_docker.rs index c27b0da6..2c1f3764 100644 --- a/packages/debmagic/src/driver/driver_docker.rs +++ b/packages/debmagic/src/driver/driver_docker.rs @@ -12,9 +12,10 @@ use serde::{Deserialize, Serialize}; use crate::driver::{ APT_MIRROR_SCRIPT, DriverType, ENVIRONMENT_DIR_IN_CONTAINER, Environment, EnvironmentDriver, EnvironmentMetadata, IsolationCapability, SignRequest, config::DriverConfig, - container_name_from_metadata, container_name_metadata, environment_fingerprint, resource_name, - run_checked, translate_path_in_container, + container_name_from_metadata, container_name_metadata, environment_fingerprint, + refresh_apt_index, resource_name, run_checked, translate_path_in_container, }; +use crate::subprocess::{self, Capture, CommandResult}; #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(default)] @@ -224,6 +225,21 @@ impl DriverDocker { ) } + /// Whether `/debmagic` in the container still resolves to the current + /// build-root inode on the host. + fn mount_is_live(&self) -> anyhow::Result { + let container_probe = crate::driver::write_mount_probe(&self.environment.root_dir)?; + let output = self.run_command( + &["test", "-f", &container_probe.to_string_lossy()], + &self.environment.root_dir, + true, + &[], + Capture::NONE, + ); + crate::driver::remove_mount_probe(&self.environment.root_dir); + Ok(output?.exit_code == 0) + } + pub fn create( environment: &Environment, driver_config: &DriverConfig, @@ -270,14 +286,22 @@ impl DriverDocker { let environment_matches = container_environment_fingerprint(&driver.container_name)? .as_deref() == Some(&desired_fingerprint); - driver.reused_environment = environment.persistent && environment_matches; let created_container; - if environment.persistent && environment_matches { - created_container = false; + let mut reuse = environment.persistent && environment_matches; + if reuse { if !driver.container_is_running()? { driver.container_start()?; } + if !driver.mount_is_live()? { + driver.container_remove_force()?; + reuse = false; + } + } + + if reuse { + driver.reused_environment = true; + created_container = false; } else { // The container may not exist; removal errors don't matter here. let _ = Command::new("docker") @@ -319,9 +343,11 @@ impl DriverDocker { // cwd is the build root (the bind mount itself), not the source dir: // create() must not assume the source tree has been staged yet. - let update_result = driver - .run_command_checked(&["apt-get", "update"], &environment.root_dir, true, &[]) - .map_err(|error| anyhow!("Error running apt-get update in container: {error}")); + // Fresh containers always update once; reused ones only when the + // configured apt update age says their index has gone stale. + let update_result = + refresh_apt_index(&driver, environment, driver_config.apt_update_age, reuse) + .map_err(|error| anyhow!("Error running apt-get update in container: {error}")); if let Err(error) = update_result { if created_container && let Err(cleanup_error) = driver.container_remove_force() { return Err(error.context(format!( @@ -365,7 +391,8 @@ impl EnvironmentDriver for DriverDocker { cwd: &Path, requires_root: bool, env_add: &[(&str, &str)], - ) -> std::io::Result { + capture: Capture, + ) -> std::io::Result { let container_path = self .translate_path_in_container(cwd) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; @@ -387,8 +414,7 @@ impl EnvironmentDriver for DriverDocker { exec_cmd.arg(&self.container_name); exec_cmd.args(cmd); - let status = exec_cmd.status()?; - Ok(status.code().unwrap_or(-1)) + subprocess::command(exec_cmd).capture(capture).run() } fn cleanup(&self) -> anyhow::Result<()> { diff --git a/packages/debmagic/src/driver/driver_lxd.rs b/packages/debmagic/src/driver/driver_lxd.rs index c1cb83a9..507fa5da 100644 --- a/packages/debmagic/src/driver/driver_lxd.rs +++ b/packages/debmagic/src/driver/driver_lxd.rs @@ -10,9 +10,10 @@ use serde::{Deserialize, Serialize}; use crate::driver::{ APT_MIRROR_SCRIPT, DriverType, ENVIRONMENT_DIR_IN_CONTAINER, Environment, EnvironmentDriver, EnvironmentMetadata, IsolationCapability, SignRequest, config::DriverConfig, - container_name_from_metadata, container_name_metadata, environment_fingerprint, resource_name, - run_checked, translate_path_in_container, + container_name_from_metadata, container_name_metadata, environment_fingerprint, + refresh_apt_index, resource_name, run_checked, translate_path_in_container, }; +use crate::subprocess::{self, Capture, CommandResult}; // The binary name differs between LXD and Incus, but everything else is shared. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] @@ -176,6 +177,52 @@ impl DriverLxd { ) } + fn source_device_name(&self) -> String { + resource_name( + "debmagic-src", + &self.environment.package_name, + &self.environment.identifier(), + ) + } + + /// Whether `/debmagic` in the container still resolves to the current + /// build-root inode on the host. + fn mount_is_live(&self) -> anyhow::Result { + let container_probe = crate::driver::write_mount_probe(&self.environment.root_dir)?; + let output = self.exec_in_container( + &["test", "-f", &container_probe.to_string_lossy()], + None, + true, + &[], + Capture::NONE, + ); + crate::driver::remove_mount_probe(&self.environment.root_dir); + Ok(output?.exit_code == 0) + } + + /// Remove and re-add the build-root disk device, re-binding it to the + /// current inode. Works on a running container. + fn remount_build_root(&self) -> anyhow::Result<()> { + let device_name = self.source_device_name(); + // Removing a device that is already absent is not an error here. + let _ = self + .lxd_cmd("config") + .args(["device", "remove", &self.container_name, &device_name]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + run_checked( + self.lxd_cmd("config") + .args(["device", "add", &self.container_name, &device_name, "disk"]) + .arg(format!("source={}", self.environment.root_dir.display())) + .arg(format!("path={}", ENVIRONMENT_DIR_IN_CONTAINER)), + &format!( + "re-mounting build root into {} container", + self.variant.binary() + ), + ) + } + fn container_environment_fingerprint(&self) -> anyhow::Result> { let output = self .lxd_cmd("config") @@ -265,6 +312,17 @@ impl DriverLxd { if !already_running { base.container_start()?; } + // The build root may have been deleted and recreated on the + // host since the container was created, leaving its disk + // device bound to the dead inode. + if !base.mount_is_live()? { + base.remount_build_root()?; + anyhow::ensure!( + base.mount_is_live()?, + "build root mount in the {} container is stale and re-mounting did not restore it", + variant.binary() + ); + } } else { if container_entry.is_some() { base.container_delete_force()?; @@ -300,11 +358,7 @@ impl DriverLxd { )?; } - let device_name = resource_name( - "debmagic-src", - &environment.package_name, - &environment.identifier(), - ); + let device_name = base.source_device_name(); run_checked( base.lxd_cmd("config") .arg("device") @@ -323,19 +377,29 @@ impl DriverLxd { )?; if matches!(environment.distro.distro, Distro::Ubuntu) { - base.exec_in_container(&["cloud-init", "status", "--wait"], None, true, &[]) - .map_err(|e| { - anyhow::anyhow!("Error waiting for cloud-init to finish: {e}") - })?; + base.exec_in_container( + &["cloud-init", "status", "--wait"], + None, + true, + &[], + Capture::NONE, + ) + .map_err(|e| anyhow::anyhow!("Error waiting for cloud-init to finish: {e}"))?; } } - // Re-run on every reuse of a persistent container too, so that a - // previous invocation that crashed before finishing this setup (or a - // long-lived incremental container with an aging package cache) - // doesn't leave `apt-get build-dep` unable to resolve anything. - base.exec_in_container_checked(&["apt-get", "update"], None, true, &[]) - .map_err(|e| anyhow::anyhow!("Error running apt-get update in container: {e}"))?; + // Fresh containers always update once; reused ones only when the + // configured apt update age says their index has gone stale — a + // previous invocation that crashed before finishing this setup (or + // a long-lived incremental container with an aging package cache) + // must not leave `apt-get build-dep` unable to resolve anything. + refresh_apt_index( + &base, + environment, + driver_config.apt_update_age, + reusing_container, + ) + .map_err(|e| anyhow::anyhow!("Error running apt-get update in container: {e}"))?; if !reusing_container { // Install the base tooling that stock images don't include. @@ -480,7 +544,8 @@ impl DriverLxd { workdir: Option<&Path>, as_root: bool, env_add: &[(&str, &str)], - ) -> std::io::Result { + capture: Capture, + ) -> std::io::Result { println!("[{}] $ {}", self.container_name, cmd.join(" ")); let mut exec_cmd = self.lxd_cmd("exec"); @@ -504,8 +569,7 @@ impl DriverLxd { exec_cmd.arg("--"); exec_cmd.args(cmd); - let status = exec_cmd.status()?; - Ok(status.code().unwrap_or(-1)) + subprocess::command(exec_cmd).capture(capture).run() } fn exec_in_container_checked( @@ -515,11 +579,12 @@ impl DriverLxd { as_root: bool, env_add: &[(&str, &str)], ) -> std::io::Result<()> { - let code = self.exec_in_container(cmd, workdir, as_root, env_add)?; - if code != 0 { + let result = self.exec_in_container(cmd, workdir, as_root, env_add, Capture::NONE)?; + if result.exit_code != 0 { return Err(std::io::Error::other(format!( - "{} exec failed with exit code {code}", - self.variant.binary() + "{} exec failed with exit code {}", + self.variant.binary(), + result.exit_code ))); } Ok(()) @@ -541,12 +606,13 @@ impl EnvironmentDriver for DriverLxd { cwd: &Path, requires_root: bool, env_add: &[(&str, &str)], - ) -> std::io::Result { + capture: Capture, + ) -> std::io::Result { let container_path = self .translate_path_in_container(cwd) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; - self.exec_in_container(cmd, Some(&container_path), requires_root, env_add) + self.exec_in_container(cmd, Some(&container_path), requires_root, env_add, capture) } fn cleanup(&self) -> anyhow::Result<()> { diff --git a/packages/debmagic/src/driver/mod.rs b/packages/debmagic/src/driver/mod.rs index 4625b7b2..25542cc9 100644 --- a/packages/debmagic/src/driver/mod.rs +++ b/packages/debmagic/src/driver/mod.rs @@ -1,16 +1,16 @@ +use anyhow::Context; +use clap::ValueEnum; +use debmagic_common::distro::DistroVersion; +use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, fmt::Debug, fs, io, path::{Path, PathBuf}, process::Command, + time::SystemTime, }; -use anyhow::Context; -use clap::ValueEnum; -use debmagic_common::distro::DistroVersion; -use serde::{Deserialize, Serialize}; - use crate::driver::{ config::{DriverConfig, DriverOverrides}, driver_bare::DriverBare, @@ -18,6 +18,8 @@ use crate::driver::{ driver_lxd::{DriverLxd, LxdVariant}, }; use crate::sign::SignTool; +use crate::subprocess::{Capture, CommandResult}; +use crate::time::{RefreshPolicy, iso_timestamp, parse_iso_timestamp}; pub mod config; pub mod driver_bare; @@ -42,6 +44,24 @@ pub fn translate_path_in_container(root_dir: &Path, path_in_source: &Path) -> io }) } +const MOUNT_PROBE_FILENAME: &str = ".debmagic-mount-probe"; + +/// Write a probe file into the build root and return its expected path inside +/// the container. A container whose `/debmagic` bind mount still points at the +/// current build-root inode sees the file; one whose mount went stale (the +/// build root was deleted and recreated on the host) does not. +pub fn write_mount_probe(root_dir: &Path) -> io::Result { + fs::write( + root_dir.join(MOUNT_PROBE_FILENAME), + b"debmagic mount probe\n", + )?; + Ok(Path::new(ENVIRONMENT_DIR_IN_CONTAINER).join(MOUNT_PROBE_FILENAME)) +} + +pub fn remove_mount_probe(root_dir: &Path) { + let _ = fs::remove_file(root_dir.join(MOUNT_PROBE_FILENAME)); +} + /// Run `cmd`, failing with `context` (and, on a clean but unsuccessful exit, /// its exit status) if it can't be spawned or exits unsuccessfully. pub fn run_checked(cmd: &mut Command, context: &str) -> anyhow::Result<()> { @@ -83,6 +103,65 @@ pub fn environment_fingerprint(parts: &[&str]) -> String { .to_string() } +/// Stamp file in a container environment's own filesystem (not the +/// bind-mounted build root, which `reset_root` wipes) holding the ISO +/// timestamp of the last successful `apt-get update` there. +pub const APT_UPDATE_STAMP: &str = "/var/lib/debmagic/apt-updated"; + +/// Bring a reused environment's apt index up to date under `policy`: check +/// the stamp file for when `apt-get update` last ran, and when the policy +/// says it has gone stale (or the stamp is missing), run the update and +/// record a fresh timestamp. Fresh environments (`reused = false`) always +/// update once. +pub fn refresh_apt_index( + driver: &dyn EnvironmentDriver, + environment: &Environment, + policy: RefreshPolicy, + reused: bool, +) -> io::Result<()> { + let last_update = if reused { + driver + .run_command( + &["cat", APT_UPDATE_STAMP], + &environment.root_dir, + true, + &[], + Capture::STDOUT, + ) + .ok() + .filter(|result| result.exit_code == 0) + .and_then(|result| result.stdout) + .and_then(|stdout| parse_iso_timestamp(&stdout)) + } else { + None + }; + if !apt_update_needed_on_reuse(policy, last_update) { + return Ok(()); + } + driver.run_command_checked(&["apt-get", "update"], &environment.root_dir, true, &[])?; + let stamp = apt_update_stamp_script(&iso_timestamp()); + driver.run_command_checked(&["sh", "-ec", &stamp], &environment.root_dir, true, &[]) +} + +/// Whether a reused environment needs `apt-get update` again under `policy`. +/// `last_update` is when its apt index was last refreshed; `None` means +/// never, which includes a missing stamp file. +pub fn apt_update_needed_on_reuse(policy: RefreshPolicy, last_update: Option) -> bool { + match policy { + RefreshPolicy::Now => true, + RefreshPolicy::Never => false, + RefreshPolicy::OlderThan(max_age) => last_update + .and_then(|last| SystemTime::now().duration_since(last).ok()) + .is_none_or(|age| age > max_age), + } +} + +/// `sh -ec` script storing `timestamp` (from [`crate::time::iso_timestamp`]) +/// in the environment's apt-update stamp file. +fn apt_update_stamp_script(timestamp: &str) -> String { + format!("mkdir -p /var/lib/debmagic && printf %s '{timestamp}' > {APT_UPDATE_STAMP}") +} + /// Metadata key under which container-based drivers store their container's /// name for later reattachment via `create_driver_from_metadata`. const CONTAINER_NAME_KEY: &str = "container_name"; @@ -108,6 +187,16 @@ pub enum DriverType { Incus, } +impl std::fmt::Display for DriverType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str( + self.to_possible_value() + .expect("no skipped variants") + .get_name(), + ) + } +} + /// Isolation an Environment actually provides for a TestRun. /// /// A ladder: none, then container, then machine. An Environment advertises @@ -201,13 +290,17 @@ pub const APT_MIRROR_SCRIPT: &str = include_str!("scripts/mirror.py"); pub trait EnvironmentDriver { fn driver_metadata(&self) -> HashMap; + /// Run `cmd` in the environment. Streams selected by `capture` are + /// collected into the returned [`CommandResult`]; the rest pass + /// through to the user's terminal. fn run_command( &self, cmd: &[&str], cwd: &Path, requires_root: bool, env_add: &[(&str, &str)], - ) -> io::Result; + capture: Capture, + ) -> io::Result; fn run_command_checked( &self, @@ -216,10 +309,11 @@ pub trait EnvironmentDriver { requires_root: bool, env_add: &[(&str, &str)], ) -> io::Result<()> { - let code = self.run_command(cmd, cwd, requires_root, env_add)?; - if code != 0 { + let result = self.run_command(cmd, cwd, requires_root, env_add, Capture::NONE)?; + if result.exit_code != 0 { return Err(io::Error::other(format!( - "Command failed with exit code: {code}" + "Command failed with exit code: {}", + result.exit_code ))); } Ok(()) @@ -267,11 +361,12 @@ impl EnvironmentDriver for Driver { cwd: &Path, requires_root: bool, env_add: &[(&str, &str)], - ) -> io::Result { + capture: Capture, + ) -> io::Result { match self { - Self::Docker(d) => d.run_command(cmd, cwd, requires_root, env_add), - Self::Bare(d) => d.run_command(cmd, cwd, requires_root, env_add), - Self::Lxd(d) => d.run_command(cmd, cwd, requires_root, env_add), + Self::Docker(d) => d.run_command(cmd, cwd, requires_root, env_add, capture), + Self::Bare(d) => d.run_command(cmd, cwd, requires_root, env_add, capture), + Self::Lxd(d) => d.run_command(cmd, cwd, requires_root, env_add, capture), } } @@ -342,18 +437,22 @@ pub fn create_driver( .as_deref() .or(driver_config.apt_mirror.as_deref()); let proposed = overrides.proposed.unwrap_or(driver_config.proposed); + let mut driver_config = driver_config.clone(); + if let Some(apt_update_age) = overrides.apt_update_age { + driver_config.apt_update_age = apt_update_age; + } match environment.driver { DriverType::Docker => Ok(Driver::Docker(DriverDocker::create( environment, - driver_config, + &driver_config, &overrides.docker, apt_mirror, proposed, )?)), DriverType::Bare => Ok(Driver::Bare(DriverBare::create( environment, - driver_config, + &driver_config, &overrides.bare, ))), DriverType::Lxd | DriverType::Incus => { @@ -364,7 +463,7 @@ pub fn create_driver( Ok(Driver::Lxd(DriverLxd::create( variant, environment, - driver_config, + &driver_config, &overrides.lxd, apt_mirror, proposed, @@ -477,6 +576,8 @@ mod tests { use debmagic_common::distro::{Distro, DistroVersion}; use std::path::PathBuf; + use std::time::Duration; + use std::time::SystemTime; #[test] fn resource_names_are_valid_stable_and_distinct() { @@ -554,4 +655,17 @@ mod tests { assert!(IsolationCapability::Container < IsolationCapability::Machine); assert!(IsolationCapability::None < IsolationCapability::Machine); } + + #[test] + fn apt_update_policy_decides_reuse_updates() { + let now = SystemTime::now(); + let hour = Duration::from_secs(60 * 60); + let older_than = RefreshPolicy::OlderThan(hour); + + assert!(apt_update_needed_on_reuse(RefreshPolicy::Now, Some(now))); + assert!(!apt_update_needed_on_reuse(RefreshPolicy::Never, None)); + assert!(!apt_update_needed_on_reuse(older_than, Some(now))); + assert!(apt_update_needed_on_reuse(older_than, Some(now - 2 * hour))); + assert!(apt_update_needed_on_reuse(older_than, None)); + } } diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 053e0691..32a01439 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -1,34 +1,51 @@ use std::env; +use std::path::Path; use std::process::ExitCode; use anyhow::Context; use clap::{CommandFactory, Parser}; +use std::str::FromStr; use crate::{ build::{build_package, build_source_package, get_shell_in_build}, build_intent::{BuildIntentInput, resolve_build_intent}, - cli::{BuildTarget, Cli, Commands, ConfigCommands}, + cli::{BuildTarget, Cli, Commands, ConfigCommands, UpstreamCommands}, config::{Config, ConfigPathStatus, resolve_set_target}, driver::{ DriverType, config::DriverOverrides, driver_bare::DriverBareConfigOverrides, driver_docker::DriverDockerConfigOverrides, driver_lxd::DriverLxdConfigOverrides, }, - package::{distro_resolve_mode_for_driver, load_package_identity, resolve_package_target}, + package::{ + distro_resolve_mode_for_driver, load_package, resolve_package_target, + validate_bare_host_target, + }, test::{TestIntentInput, TestOutcome, resolve_test_intent, run_test}, }; pub mod build; pub mod build_intent; +pub mod changelog; +pub mod changes; pub mod cli; pub mod config; +pub mod control; pub mod driver; pub mod output; pub mod package; +pub mod requests; pub mod sign; +pub mod subprocess; pub mod test; +pub mod time; +pub mod upload; +pub mod upstream; fn main() -> ExitCode { - match run() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to build the tokio runtime"); + match runtime.block_on(run()) { Ok(code) => code, Err(error) => { eprintln!("{error:?}"); @@ -37,20 +54,30 @@ fn main() -> ExitCode { } } -fn run() -> anyhow::Result { +async fn run() -> anyhow::Result { let cli = Cli::parse(); output::init_color(cli.color); let current_dir = env::current_dir()?; match &cli.command { Commands::Build(args) => { - let (build_args, debug_symbols, is_source) = match &args.target { - BuildTarget::Binary(binary_args) => { - (&binary_args.build, binary_args.debug_symbols, false) - } - BuildTarget::Source(source_args) => (&source_args.build, None, true), + let (build_args, debug_symbols, is_source, include_orig, run_test) = match &args.target + { + BuildTarget::Binary(binary_args) => ( + &binary_args.build, + binary_args.debug_symbols, + false, + None, + binary_args.test, + ), + BuildTarget::Source(source_args) => ( + &source_args.build, + None, + true, + Some(source_args.include_orig), + Some(false), + ), }; - let config_driver = Config::load( build_args.common.source_dir.as_deref(), cli.config.as_deref(), @@ -78,6 +105,7 @@ fn run() -> anyhow::Result { persistent: build_args.persistent, incremental: build_args.incremental, debug_symbols, + test: run_test, sign: build_args.sign, sign_key: build_args.sign_key.clone(), sign_tool: build_args.sign_tool, @@ -90,6 +118,7 @@ fn run() -> anyhow::Result { driver_overrides: DriverOverrides { apt_mirror: build_args.apt_mirror.clone(), proposed: build_args.proposed, + apt_update_age: build_args.apt_update_age, docker: DriverDockerConfigOverrides { base_image: build_args.docker.base_image.clone(), }, @@ -113,10 +142,50 @@ fn run() -> anyhow::Result { .context("failed to determine package target")?; if is_source { - build_source_package(&intent, &target) + let include_orig = match include_orig { + Some(mode) => upload::orig::decide_orig_upload(mode, &intent.source_dir)?, + None => true, + }; + build_source_package(&intent, &target, &build_args.changes_options, include_orig) + .await .context("Building the source package failed")?; } else { - build_package(&intent, &target).context("Building the package failed")?; + let mut target = target; + if intent.driver == DriverType::Bare && !build_args.bare_ignore_release { + target.distro = + validate_bare_host_target(&target.distro, Path::new("/etc/os-release")) + .context( + "host's /etc/os-release does not match the build target distro", + )?; + } + build_package(&intent, &target, &build_args.changes_options) + .context("Building the package failed")?; + } + + if let Some(spec) = &build_args.upload { + let upload_target = + upload::resolve_target(spec, Some(&intent.config.upload.targets))?; + let identity = load_package(&intent.source_dir)?; + let changes_file = crate::sign::find_changes_file( + identity.name(), + &identity.version().to_string(), + &intent.output_dir, + )?; + crate::output::stage(&format!("Uploading to {}", upload_target.name)); + upload::upload_changes( + &upload_target, + spec, + &changes_file, + false, + false, + Some(identity.version().upstream_version()), + ) + .with_context(|| { + format!( + "uploading {} to target '{spec}' failed", + changes_file.display() + ) + })?; } } Commands::Shell(args) => { @@ -124,7 +193,7 @@ fn run() -> anyhow::Result { let source_dir = std::path::absolute(source_dir).context("resolving source dir failed")?; let config = Config::load(Some(&source_dir), cli.config.as_deref())?; - let identity = load_package_identity(&source_dir)?; + let identity = load_package(&source_dir)?; get_shell_in_build(&config, &identity)?; } Commands::Test(args) => { @@ -142,6 +211,7 @@ fn run() -> anyhow::Result { driver_overrides: DriverOverrides { apt_mirror: args.apt_mirror.clone(), proposed: args.proposed, + apt_update_age: args.apt_update_age, docker: DriverDockerConfigOverrides { base_image: args.docker.base_image.clone(), }, @@ -163,6 +233,77 @@ fn run() -> anyhow::Result { Commands::Check(_args) => { println!("Check subcommand! - not implemented"); } + Commands::Upload(args) => { + let source_dir = args.common.source_dir.as_deref().unwrap_or(¤t_dir); + let source_dir = + std::path::absolute(source_dir).context("resolving source dir failed")?; + let config = Config::load(Some(&source_dir), cli.config.as_deref())?; + + let mut upload_target = + upload::resolve_target(&args.target, Some(&config.upload.targets))?; + upload_target.apply_overrides(&upload::UploadOverrides { + method: args.method, + server: args.server.clone(), + incoming: args.incoming.clone(), + login: args.login.clone(), + port: args.port, + }); + + let identity = load_package(&source_dir)?; + let changes_file = match &args.changes { + Some(file) => { + std::path::absolute(file).context("resolving the changes file failed")? + } + None => { + let output_dir = std::path::absolute(source_dir.join(&config.output_dir)) + .context("resolving output dir failed")?; + crate::sign::find_changes_file( + identity.name(), + &identity.version().to_string(), + &output_dir, + )? + } + }; + + let upstream_version = identity.version().upstream_version().to_string(); + + if let Some(mode) = args.include_orig { + let include = upload::orig::decide_orig_upload(mode, &source_dir)?; + let sign_options = sign::SignOptions { + key: config.sign.key.clone(), + tool: config.sign.tool, + sign_command: config.sign.sign_command.clone(), + verify_command: config.sign.verify_command.clone(), + }; + upload::orig::changes_include_orig( + &changes_file, + include, + &identity, + &sign_options, + )?; + } + + crate::output::stage(&format!( + "Uploading {} to {}", + changes_file.display(), + upload_target.name + )); + upload::upload_changes( + &upload_target, + &args.target, + &changes_file, + args.no_hooks, + args.force, + Some(&upstream_version), + ) + .with_context(|| { + format!( + "uploading {} to target '{}' failed", + changes_file.display(), + args.target + ) + })?; + } Commands::Sign(args) => { let source_dir = args.common.source_dir.as_deref().unwrap_or(¤t_dir); let source_dir = @@ -176,7 +317,7 @@ fn run() -> anyhow::Result { config.sign.tool = tool; } if let Some(command) = &args.sign_command { - config.sign.command = Some(command.clone()); + config.sign.sign_command = Some(command.clone()); } if let Some(notify) = args.sign_notify { config.sign.notify = notify; @@ -185,7 +326,8 @@ fn run() -> anyhow::Result { let options = sign::SignOptions { key: config.sign.key.clone(), tool: config.sign.tool, - command: config.sign.command.clone(), + sign_command: config.sign.sign_command.clone(), + verify_command: config.sign.verify_command.clone(), }; let file = match &args.file { @@ -193,7 +335,7 @@ fn run() -> anyhow::Result { std::path::absolute(file).context("resolving the file to sign failed")? } None => { - let identity = load_package_identity(&source_dir)?; + let identity = load_package(&source_dir)?; // -o wins; else the config value, relative to the package root. let output_dir = match &args.output_dir { Some(dir) => { @@ -203,8 +345,8 @@ fn run() -> anyhow::Result { .context("resolving output dir failed")?, }; sign::find_changes_file( - &identity.name, - &identity.version.to_string(), + identity.name(), + &identity.version().to_string(), &output_dir, )? } @@ -237,6 +379,8 @@ fn run() -> anyhow::Result { } let config = Config::new(&paths)?; + let effective_driver = config.driver.default.unwrap_or(DriverType::Bare); + eprintln!("debmagic: using driver: {effective_driver} (cfg: driver.default)"); print!("{}", toml::to_string_pretty(&config)?); } ConfigCommands::Get(get_args) => { @@ -275,6 +419,180 @@ fn run() -> anyhow::Result { eprintln!("debmagic: written to {}", target.path.display()); } }, + Commands::Upstream(args) => match &args.command { + UpstreamCommands::List(list_args) => { + let source_dir = list_args + .common + .source_dir + .as_deref() + .unwrap_or(¤t_dir); + let source_dir = + std::path::absolute(source_dir).context("resolving source dir failed")?; + let sources = upstream::query::load_watch(&source_dir)?; + let identity = load_package(&source_dir)?; + + let mut newest: Option = None; + for source in &sources { + if let Some(reason) = &source.untrackable { + println!("debmagic: skipping untrackable source: {reason}"); + continue; + } + let candidates = upstream::query::query_source(source, identity.name()).await?; + let current = upstream::query::current_upstream_version( + source, + &identity.version().to_string(), + )?; + let current_version = + debmagic_common::debian::version::PackageVersion::from_str(¤t) + .map_err(|_| anyhow::anyhow!("invalid current version: {current}"))?; + let newer: Vec<_> = candidates + .iter() + .filter(|c| { + debmagic_common::debian::version::PackageVersion::from_str(&c.version) + .is_ok_and(|v| v > current_version) + }) + .collect(); + println!("debmagic: current upstream version: {current}"); + if list_args.all { + for candidate in &candidates { + println!(" {}", candidate.version); + } + } else { + let limit = list_args.previous.unwrap_or(1); + for candidate in newer.iter().take(limit) { + println!(" {} (newer)", candidate.version); + } + } + newest = candidates.first().map(|c| c.version.clone()); + } + if newest.is_none() { + println!("debmagic: no candidates found"); + } + } + UpstreamCommands::Switch(switch_args) => { + let source_dir = switch_args + .common + .source_dir + .as_deref() + .unwrap_or(¤t_dir); + let source_dir = + std::path::absolute(source_dir).context("resolving source dir failed")?; + let sources = upstream::query::load_watch(&source_dir)?; + let identity = load_package(&source_dir)?; + let mut config = Config::load(Some(&source_dir), cli.config.as_deref())?; + let output_dir = source_dir.join(&config.output_dir); + + // the main source (no Component field) drives the version; + // component sources contribute their own tarballs + let main_source = sources + .iter() + .find(|s| s.untrackable.is_none() && s.component.is_none()) + .context("no usable main watch source for {}")?; + let candidate = if switch_args.version == "latest" { + // discovery needs the listing + let candidates = + upstream::query::query_source(main_source, identity.name()).await?; + candidates.first().cloned() + } else { + // a concrete version: construct the URL from the watch + // pattern directly; only fall back to scraping when the + // pattern is not invertible or the URL does not exist. + // an existing orig tarball hints at the extension first. + let existing_orig = debmagic_common::changes::find_orig_in_dir( + &output_dir, + identity.name(), + identity.version().upstream_version(), + ) + .or_else(|| { + source_dir.parent().and_then(|parent| { + debmagic_common::changes::find_orig_in_dir( + parent, + identity.name(), + identity.version().upstream_version(), + ) + }) + }); + match upstream::query::resolve_concrete( + main_source, + identity.name(), + &switch_args.version, + existing_orig.as_deref(), + ) + .await + { + Ok(Some(candidate)) => Some(candidate), + Ok(None) | Err(_) => { + let candidates = + upstream::query::query_source(main_source, identity.name()).await?; + upstream::query::find_candidate(&candidates, &switch_args.version) + .cloned() + } + } + }; + let Some(candidate) = candidate else { + if switch_args.version == "latest" { + anyhow::bail!("no upstream candidates found for {}", identity.name()); + } + anyhow::bail!( + "upstream version {} not found for {}; run 'debmagic upstream list' to see the available versions", + switch_args.version, + identity.name() + ); + }; + + let package = crate::package::load_package(&source_dir)?; + let repack = upstream::repack::load_repack_config(&package, main_source)?; + let verify = config.upstream.verify_signatures && !switch_args.no_signature_check; + if let Some(command) = &switch_args.verify_command { + config.sign.verify_command = Some(command.clone()); + } + let sign_options = sign::SignOptions { + key: config.sign.key.clone(), + tool: config.sign.tool, + sign_command: config.sign.sign_command.clone(), + verify_command: config.sign.verify_command.clone(), + }; + let options = upstream::switch::SwitchOptions { + repack: &repack, + output_dir: &output_dir, + orig_tarball_config: Some(&config.orig_tarball), + verify_signatures: verify, + sign_options: &sign_options, + dry_run: switch_args.dry_run, + }; + upstream::switch::switch(&source_dir, main_source, &candidate, &options).await?; + + // MUT: switch each component source to the same version + for source in &sources { + if source.untrackable.is_some() || source.component.is_none() { + continue; + } + let candidates = upstream::query::query_source(source, identity.name()).await?; + let Some(component_candidate) = candidates + .iter() + .find(|c| c.version == candidate.version) + .or_else(|| candidates.first()) + else { + anyhow::bail!( + "no candidates for component {} at version {}", + source.component.as_deref().unwrap_or_default(), + candidate.version + ); + }; + let repack = upstream::repack::load_repack_config(&package, source)?; + let options = upstream::switch::SwitchOptions { + repack: &repack, + output_dir: &output_dir, + orig_tarball_config: Some(&config.orig_tarball), + verify_signatures: verify, + sign_options: &sign_options, + dry_run: switch_args.dry_run, + }; + upstream::switch::switch(&source_dir, source, component_candidate, &options) + .await?; + } + } + }, Commands::Version {} => { let cmd = Cli::command(); println!("{}", cmd.render_version()); diff --git a/packages/debmagic/src/package.rs b/packages/debmagic/src/package.rs index d2022188..dcdf777e 100644 --- a/packages/debmagic/src/package.rs +++ b/packages/debmagic/src/package.rs @@ -1,24 +1,16 @@ use std::collections::HashMap; -use std::path::{Path, PathBuf}; +use std::path::Path; -use anyhow::anyhow; -use debmagic_common::debian::version::PackageVersion; +use anyhow::{Context, anyhow, bail}; use debmagic_common::distro::{Distro, DistroVersion, get_distro_version}; +use debmagic_common::package::{FileReader, Location, SourcePackage}; use crate::driver::DriverType; -/// Who/what is being built, as read from the source tree changelog. -#[derive(Debug, Clone)] -pub struct PackageIdentity { - pub name: String, - pub version: PackageVersion, - pub source_dir: PathBuf, -} - -/// A [`PackageIdentity`] plus the chosen [`DistroVersion`] for a build run. +/// A [`Package`] plus the chosen [`DistroVersion`] for a build run. #[derive(Debug, Clone)] pub struct PackageTarget { - pub identity: PackageIdentity, + pub package: SourcePackage, pub distro: DistroVersion, } @@ -31,54 +23,21 @@ pub enum DistroResolveMode<'a> { /// e.g. `driver.docker.base_images` — used in error messages. config_key: &'a str, }, - /// Bare: custom suites require a host `/etc/os-release` codename match; - /// built-in Debian/Ubuntu also require family (`ID`) match. - Bare { - /// Usually `/etc/os-release`; overridable in tests. - os_release_path: &'a Path, - }, -} - -struct ChangelogPackage { - identity: PackageIdentity, - /// Raw distribution names from the changelog entry (not looked up yet). - changelog_distros: Vec, -} - -fn read_changelog_package(dir: &Path) -> anyhow::Result { - let changelog_file = dir.join("debian").join("changelog"); - let changelog_contents = std::fs::read_to_string(changelog_file)?; - let changelog: debian_changelog::ChangeLog = changelog_contents.parse()?; - - let first_entry = changelog - .into_iter() - .next() - .ok_or(anyhow!("changelog is empty"))?; - - let name = first_entry - .package() - .ok_or(anyhow!("empty package name in changelog entry"))?; - let version = first_entry - .version() - .ok_or(anyhow!("no package version in changelog entry")) - .map(|v| PackageVersion::new(v.epoch, v.upstream_version, v.debian_revision))?; - - let changelog_distros = first_entry - .distributions() - .ok_or(anyhow!("no distribution specified in changelog entry"))?; - - Ok(ChangelogPackage { - identity: PackageIdentity { - name, - version, - source_dir: dir.to_path_buf(), - }, - changelog_distros, - }) + /// Bare driver: the system running debmagic is used directly. + Bare, } -pub fn load_package_identity(dir: &Path) -> anyhow::Result { - Ok(read_changelog_package(dir)?.identity) +/// Read a source tree's `debian/` metadata into the common [`Package`] model. +/// The package reads its files lazily through a closure over the source +/// dir, so only the metadata actually requested is ever read. +pub fn load_package(dir: &Path) -> anyhow::Result { + let source_dir = dir.to_path_buf(); + let reader_dir = source_dir.clone(); + let reader: FileReader = Box::new(move |name| { + let path = reader_dir.join("debian").join(name); + std::fs::read_to_string(&path).with_context(|| format!("failed to read {}", path.display())) + }); + SourcePackage::from_reader(std::rc::Rc::new(reader), Location::SourceDir(source_dir)) } /// Resolve package identity and target distro from a source tree. @@ -92,12 +51,9 @@ pub fn resolve_package_target( explicit_distro: Option<&str>, mode: DistroResolveMode<'_>, ) -> anyhow::Result { - let parsed = read_changelog_package(dir)?; - let distro = select_distro_version(&parsed.changelog_distros, explicit_distro, mode)?; - Ok(PackageTarget { - identity: parsed.identity, - distro, - }) + let package = load_package(dir)?; + let distro = select_distro_version(package.distributions(), explicit_distro, mode)?; + Ok(PackageTarget { package, distro }) } /// Pick the resolve mode for the active Driver from its config maps. @@ -115,9 +71,7 @@ pub fn distro_resolve_mode_for_driver<'a>( base_images: lxd_base_images, config_key: "driver.lxd.base_images", }, - DriverType::Bare => DistroResolveMode::Bare { - os_release_path: Path::new("/etc/os-release"), - }, + DriverType::Bare => DistroResolveMode::Bare, } } @@ -200,9 +154,6 @@ fn lookup_distro(name: &str, mode: DistroResolveMode<'_>) -> anyhow::Result) -> anyhow::Result custom_distro_from_base_images(name, base_images, config_key), - DistroResolveMode::Bare { os_release_path } => { - let os = read_os_release(os_release_path)?; - let host_codename = os.codename().ok_or_else(|| { - anyhow!("host {} has no VERSION_CODENAME", os_release_path.display()) - })?; - if host_codename != name { - return Err(anyhow!( - "unknown distro codename '{name}' does not match host VERSION_CODENAME \ - '{host_codename}'. For Bare builds of non-Debian/Ubuntu suites, the host \ - codename must match; for container Drivers, declare the suite in \ - base_images (e.g. driver.docker.base_images = {{ \"yocto:{name}\" = \"\" }})" - )); - } - let family = os - .id - .ok_or_else(|| anyhow!("host {} has no ID", os_release_path.display()))?; - Ok(DistroVersion::custom(Distro::from(family), name)) - } + DistroResolveMode::Bare => Ok(DistroVersion::custom(Distro::from(name), name)), } } -fn check_bare_os_release_for_builtin( - os_release_path: &Path, +/// A binary build on the Bare driver runs directly on the host, so the host +/// must provide the target environment: built-in Debian/Ubuntu targets need +/// a matching `ID` and codename in the host `os-release`, custom suites a +/// matching codename (their placeholder family is anchored to the host `ID`). +/// Source builds skip this check — their artifacts are distro-independent. +pub fn validate_bare_host_target( target: &DistroVersion, -) -> anyhow::Result<()> { - if matches!(target.distro, Distro::Custom(_)) { - return Ok(()); - } + os_release_path: &Path, +) -> anyhow::Result { let os = read_os_release(os_release_path)?; - let host_codename = os - .codename() - .ok_or_else(|| anyhow!("host {} has no VERSION_CODENAME", os_release_path.display()))?; let host_id = os .id .as_deref() .ok_or_else(|| anyhow!("host {} has no ID", os_release_path.display()))?; + let host_codename = os.codename().ok_or_else(|| { + anyhow!( + "host {} has no VERSION_CODENAME, cannot verify it for a bare binary build", + os_release_path.display() + ) + })?; + + if let Distro::Custom(_) = target.distro { + if target.codename != host_codename { + bail!( + "bare binary build targets custom suite '{}' but host VERSION_CODENAME is \ + '{host_codename}'; the host must match, or use a container driver with the \ + suite declared in base_images", + target.codename + ); + } + return Ok(DistroVersion::custom( + Distro::from(host_id), + &target.codename, + )); + } if host_id != target.distro.as_str() { - return Err(anyhow!( - "Bare build targets {} but host {} ID is '{}'", + bail!( + "bare binary build targets {} but host {} ID is '{host_id}'", target.distro, - os_release_path.display(), - host_id - )); + os_release_path.display() + ); } - if host_codename != target.codename { - return Err(anyhow!( - "Bare build targets {} {} but host VERSION_CODENAME is '{}'", + if target.codename != host_codename { + bail!( + "bare binary build targets {} {} but host VERSION_CODENAME is '{host_codename}'", target.distro, - target.codename, - host_codename - )); + target.codename + ); } - Ok(()) + Ok(target.clone()) } #[derive(Debug, Clone, PartialEq, Eq)] @@ -320,48 +272,34 @@ fn select_distro_version( explicit_distro: Option<&str>, mode: DistroResolveMode<'_>, ) -> anyhow::Result { - match (changelog_distros.len(), explicit_distro) { - (0, _) => Err(anyhow!("changelog contains no distributions")), - (1, None) => lookup_distro(&changelog_distros[0], mode), - (1, Some(explicit)) => { - let from_changelog = lookup_distro(&changelog_distros[0], mode)?; - let from_explicit = lookup_distro(explicit, mode)?; - if from_changelog == from_explicit { - Ok(from_explicit) - } else { - Err(anyhow!( - "explicit distro version '{}' conflicts with distribution specified in changelog '{}'", - explicit, - changelog_distros[0] - )) - } - } - (_, None) => Err(anyhow!( - "changelog contains multiple distributions ({}), please specify which one to build for with --distro", + let Some(explicit) = explicit_distro else { + return match changelog_distros { + [] => bail!("changelog contains no distributions"), + [single] => lookup_distro(single, mode), + many => bail!( + "changelog contains multiple distributions ({}), please specify which one to build for with --distro", + many.join(", ") + ), + }; + }; + + let resolved = lookup_distro(explicit, mode)?; + let in_changelog = changelog_distros + .iter() + .any(|name| name == explicit || lookup_distro(name, mode).is_ok_and(|d| d == resolved)); + if !in_changelog { + println!( + "debmagic: building for '{explicit}', changelog targets {}", changelog_distros.join(", ") - )), - (_, Some(explicit)) => { - let from_explicit = lookup_distro(explicit, mode)?; - let matched = changelog_distros.iter().any(|name| { - lookup_distro(name, mode) - .is_ok_and(|from_changelog| from_changelog == from_explicit) - }); - if matched { - Ok(from_explicit) - } else { - Err(anyhow!( - "explicit distro version '{}' not found in changelog distributions: {}", - explicit, - changelog_distros.join(", ") - )) - } - } + ); } + Ok(resolved) } #[cfg(test)] mod tests { use super::*; + use std::path::PathBuf; fn test_package_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -385,13 +323,13 @@ mod tests { } #[test] - fn load_package_identity_from_changelog() -> anyhow::Result<()> { + fn load_package_from_changelog() -> anyhow::Result<()> { let dir = test_package_dir(); - let identity = load_package_identity(&dir)?; + let package = load_package(&dir)?; - assert_eq!(identity.name, "test-package"); - assert_eq!(identity.version.version(), "1.2.4-1"); - assert_eq!(identity.source_dir, dir); + assert_eq!(package.name(), "test-package"); + assert_eq!(package.version().version(), "1.2.4-1"); + assert_eq!(package.source_dir()?, dir); Ok(()) } @@ -444,7 +382,7 @@ mod tests { Some("unstable"), docker_mode(&empty), )?; - assert_eq!(target.identity.name, "test-package"); + assert_eq!(target.package.name(), "test-package"); assert_eq!(target.distro.codename, "unstable"); assert_eq!(target.distro.distro, Distro::Debian); Ok(()) @@ -469,17 +407,12 @@ mod tests { } #[test] - fn select_distro_version_single_conflicting_explicit() { + fn select_distro_version_single_conflicting_explicit_overrides() -> anyhow::Result<()> { let empty = HashMap::new(); - let result = - select_distro_version(&["forky".to_string()], Some("duke"), docker_mode(&empty)); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("conflicts with distribution specified in changelog") - ); + let distro = + select_distro_version(&["forky".to_string()], Some("duke"), docker_mode(&empty))?; + assert_eq!(distro.codename, "duke"); + Ok(()) } #[test] @@ -512,20 +445,15 @@ mod tests { } #[test] - fn select_distro_version_multiple_explicit_invalid() { + fn select_distro_version_multiple_explicit_not_in_changelog_overrides() -> anyhow::Result<()> { let empty = HashMap::new(); - let result = select_distro_version( + let distro = select_distro_version( &["forky".to_string(), "duke".to_string()], Some("trixie"), docker_mode(&empty), - ); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("not found in changelog distributions") - ); + )?; + assert_eq!(distro.codename, "trixie"); + Ok(()) } #[test] @@ -599,42 +527,61 @@ UBUNTU_CODENAME=ignored } #[test] - fn bare_builtin_requires_id_and_codename_match() -> anyhow::Result<()> { + fn bare_mode_resolves_any_suite_without_host_check() -> anyhow::Result<()> { + let distro = select_distro_version(&["trixie".to_string()], None, DistroResolveMode::Bare)?; + assert_eq!(distro.distro, Distro::Debian); + assert_eq!(distro.codename, "trixie"); + + let distro = + select_distro_version(&["kirkstone".to_string()], None, DistroResolveMode::Bare)?; + assert_eq!(distro.distro, Distro::Custom("kirkstone".into())); + assert_eq!(distro.codename, "kirkstone"); + Ok(()) + } + + #[test] + fn bare_binary_build_requires_id_and_codename_match() -> anyhow::Result<()> { let os_path = std::env::temp_dir().join(format!( "debmagic-os-release-builtin-{}", std::process::id() )); + let target = DistroVersion::new(Distro::Debian, "trixie", "13"); + std::fs::write(&os_path, "ID=debian\nVERSION_CODENAME=bookworm\n")?; - let mode = DistroResolveMode::Bare { - os_release_path: &os_path, - }; - let err = select_distro_version(&["trixie".to_string()], None, mode).unwrap_err(); - assert!(err.to_string().contains("VERSION_CODENAME")); + let err = validate_bare_host_target(&target, &os_path).unwrap_err(); + assert!(err.to_string().contains("VERSION_CODENAME is 'bookworm'")); std::fs::write(&os_path, "ID=ubuntu\nVERSION_CODENAME=trixie\n")?; - let err = select_distro_version(&["trixie".to_string()], None, mode).unwrap_err(); + let err = validate_bare_host_target(&target, &os_path).unwrap_err(); assert!(err.to_string().contains("ID is 'ubuntu'")); std::fs::write(&os_path, "ID=debian\nVERSION_CODENAME=trixie\n")?; - let distro = select_distro_version(&["trixie".to_string()], None, mode)?; + let distro = validate_bare_host_target(&target, &os_path)?; assert_eq!(distro.distro, Distro::Debian); assert_eq!(distro.codename, "trixie"); + + std::fs::write(&os_path, "ID=gentoo\n")?; + let err = validate_bare_host_target(&target, &os_path).unwrap_err(); + assert!(err.to_string().contains("no VERSION_CODENAME")); let _ = std::fs::remove_file(&os_path); Ok(()) } #[test] - fn bare_custom_matches_codename_only() -> anyhow::Result<()> { + fn bare_binary_build_custom_suite_anchors_family_to_host() -> anyhow::Result<()> { let os_path = std::env::temp_dir().join(format!("debmagic-os-release-custom-{}", std::process::id())); + let target = DistroVersion::custom(Distro::from("kirkstone"), "kirkstone"); + std::fs::write(&os_path, "ID=yocto\nVERSION_CODENAME=kirkstone\n")?; - let mode = DistroResolveMode::Bare { - os_release_path: &os_path, - }; - let distro = select_distro_version(&["kirkstone".to_string()], None, mode)?; + let distro = validate_bare_host_target(&target, &os_path)?; assert_eq!(distro.distro, Distro::Custom("yocto".into())); assert_eq!(distro.codename, "kirkstone"); assert_eq!(distro.version, ""); + + std::fs::write(&os_path, "ID=yocto\nVERSION_CODENAME=other\n")?; + let err = validate_bare_host_target(&target, &os_path).unwrap_err(); + assert!(err.to_string().contains("custom suite 'kirkstone'")); let _ = std::fs::remove_file(&os_path); Ok(()) } diff --git a/packages/debmagic/src/requests.rs b/packages/debmagic/src/requests.rs new file mode 100644 index 00000000..2d6a54a6 --- /dev/null +++ b/packages/debmagic/src/requests.rs @@ -0,0 +1,123 @@ +use std::path::Path; + +use anyhow::Context; +use futures_util::StreamExt; + +/// The HTTP client for all external requests: no overall timeout +/// (tarballs can be huge and slow), but a connect timeout so dead +/// hosts fail fast. +fn http_client() -> reqwest::Client { + reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(30)) + .build() + .expect("reqwest client with a custom timeout") +} + +pub async fn http_get(url: &str) -> anyhow::Result { + let response = http_client() + .get(url) + .send() + .await + .with_context(|| format!("requesting {url} failed"))?; + if !response.status().is_success() { + anyhow::bail!("GET {url} returned {}", response.status()); + } + let body = response + .text() + .await + .with_context(|| format!("reading the response of {url} failed"))?; + Ok(body) +} + +/// Whether `url` exists, via a HEAD request. +pub async fn http_exists(url: &str) -> bool { + http_client() + .head(url) + .send() + .await + .is_ok_and(|response| response.status().is_success()) +} + +/// Stream `url` to `destination` without buffering it in memory. +pub async fn http_download(url: &str, destination: &Path) -> anyhow::Result<()> { + let response = http_client() + .get(url) + .send() + .await + .with_context(|| format!("requesting {url} failed"))?; + if !response.status().is_success() { + anyhow::bail!("GET {url} returned {}", response.status()); + } + let mut file = std::fs::File::create(destination) + .with_context(|| format!("failed to create {}", destination.display()))?; + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.with_context(|| format!("reading the response of {url} failed"))?; + std::io::Write::write_all(&mut file, &chunk) + .with_context(|| format!("failed to write {}", destination.display()))?; + } + Ok(()) +} + +pub async fn ftp_get(url: &str) -> anyhow::Result { + let url = url.to_string(); + tokio::task::spawn_blocking(move || { + let output = curl(&url, &[])?; + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + }) + .await + .context("ftp fetch task panicked")? +} + +/// Download `url` via curl into `destination`, without buffering it +/// in memory. +pub async fn ftp_download(url: &str, destination: &Path) -> anyhow::Result<()> { + let url = url.to_string(); + let destination = destination.to_string_lossy().into_owned(); + tokio::task::spawn_blocking(move || { + curl(&url, &["--output", &destination])?; + Ok(()) + }) + .await + .context("ftp download task panicked")? +} + +/// Run curl on `url` with extra args, failing with an actionable +/// message when curl is missing. +fn curl(url: &str, args: &[&str]) -> anyhow::Result { + let output = std::process::Command::new("curl") + .arg("--silent") + .arg("--show-error") + .arg("--fail") + .args(args) + .arg(url) + .output() + .map_err(|e| { + if e.kind() == std::io::ErrorKind::NotFound { + anyhow::anyhow!( + "curl is required for ftp sources like {url} but is not installed; \ + install the curl package" + ) + } else { + anyhow::anyhow!(e).context(format!("failed to run curl for {url}")) + } + })?; + if !output.status.success() { + anyhow::bail!( + "curl fetch of {url} failed (exit status: {}): {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(output) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_http_get_rejects_bad_url() { + assert!(http_get("not-a-url").await.is_err()); + } +} diff --git a/packages/debmagic/src/sign.rs b/packages/debmagic/src/sign.rs index 655e1fbf..de289f05 100644 --- a/packages/debmagic/src/sign.rs +++ b/packages/debmagic/src/sign.rs @@ -8,19 +8,18 @@ use std::{ path::{Path, PathBuf}, - process::{Command, Stdio}, + process::Command, }; use anyhow::{Context, bail}; -use deb822_lossless::{Deb822, Paragraph}; +use deb822_lossless::Paragraph; use debian_control::pgp; -use md5::Md5; use serde::{Deserialize, Serialize}; -use sha1::Sha1; -use sha2::{Digest, Sha256}; +use crate::control::{child_filename, fixup_checksums, read_control, write_control}; use crate::driver::SignRequest; use crate::output::notify_send_bell; +use crate::subprocess::Capture; /// Which OpenPGP implementation performs the signing. #[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, clap::ValueEnum)] @@ -36,6 +35,115 @@ pub enum SignTool { Custom, } +/// Verify a detached signature `signature` over `file` against the +/// keyring `keyring` (an exported OpenPGP key, like +/// `debian/upstream/signing-key.asc`), using the configured backend. +/// With `tool = "custom"`, `sign.command` runs with the `{file}`, +/// `{signature}` and `{keyring}` placeholders substituted; a command +/// without `{signature}` gets the signature path appended as the last +/// argument. A non-zero exit means the verification failed. +pub fn verify_signature( + options: &SignOptions, + file: &Path, + signature: &Path, + keyring: &Path, +) -> anyhow::Result<()> { + let run = |cmd: Command| { + let result = crate::subprocess::command(cmd) + .capture(Capture::ALL) + .run() + .with_context(|| { + format!( + "failed to run the signature verification of {}", + file.display() + ) + })?; + if result.exit_code != 0 { + bail!( + "signature verification of {} failed:\n{}", + file.display(), + result.stderr.as_deref().unwrap_or_default().trim() + ); + } + Ok(()) + }; + + match options.tool { + SignTool::Gpg => { + let mut cmd = Command::new("gpg"); + cmd.args(["--no-default-keyring", "--keyring"]) + .arg(keyring) + .arg("--verify") + .arg(signature) + .arg(file); + run(cmd) + } + SignTool::Sequoia => { + let mut cmd = Command::new("sq"); + cmd.arg("verify") + .arg("--keyring") + .arg(keyring) + .arg(signature) + .arg(file); + run(cmd) + } + SignTool::Custom => { + let command = options + .verify_command + .as_deref() + .or(options.sign_command.as_deref()) + .context("sign.tool = \"custom\" requires sign.verify_command (or sign.sign_command) to be set")?; + let mut parts = command.split_whitespace(); + let program = parts.next().context("the custom verify command is empty")?; + let mut cmd = Command::new(program); + let mut has_signature = false; + for part in parts { + has_signature |= part.contains("{signature}"); + cmd.arg(substitute_verify_placeholders( + part, file, signature, keyring, + )?); + } + if !has_signature { + cmd.arg(signature); + } + run(cmd) + } + } +} + +/// Substitute the `{file}`, `{signature}` and `{keyring}` placeholders in +/// one custom verify-command argument, rejecting unknown or unterminated +/// ones. +fn substitute_verify_placeholders( + arg: &str, + file: &Path, + signature: &Path, + keyring: &Path, +) -> anyhow::Result { + let mut out = String::new(); + let mut rest = arg; + while let Some(start) = rest.find('{') { + out.push_str(&rest[..start]); + let after = &rest[start + 1..]; + let end = after + .find('}') + .with_context(|| format!("unterminated '{{' in the verify command argument '{arg}'"))?; + let name = &after[..end]; + let value = match name { + "file" => file.display().to_string(), + "signature" => signature.display().to_string(), + "keyring" => keyring.display().to_string(), + _ => bail!( + "unknown placeholder '{{{name}}}' in the verify command (supported: {{file}}, {{signature}}, {{keyring}})" + ), + }; + out.push_str(&value); + rest = &after[end + 1..]; + } + out.push_str(rest); + Ok(out) +} + /// How to sign: which key to use, and which program does the OpenPGP work. #[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default)] @@ -50,7 +158,13 @@ pub struct SignOptions { /// without a shell; `{file}`, `{key}` and `{email}` placeholders are /// substituted, and if no `{file}` is given the file path is appended /// as the last argument. The clearsigned result is read from stdout. - pub command: Option, + pub sign_command: Option, + /// Custom verification command when `tool` is [`SignTool::Custom`], + /// used by signature checks (e.g. `upstream switch`). Run without a + /// shell; `{file}`, `{signature}` and `{keyring}` placeholders are + /// substituted, and if no `{signature}` is given the signature path is + /// appended as the last argument. Falls back to `sign_command`. + pub verify_command: Option, } /// Sign the `.changes` file (and its `.dsc`/`.buildinfo` children) on the @@ -59,7 +173,8 @@ pub fn sign_changes(request: &SignRequest) -> anyhow::Result<()> { let options = SignOptions { key: request.sign_key.map(str::to_string), tool: request.sign_tool, - command: request.sign_command.map(str::to_string), + sign_command: request.sign_command.map(str::to_string), + verify_command: None, }; sign_file( request.changes_file, @@ -69,127 +184,6 @@ pub fn sign_changes(request: &SignRequest) -> anyhow::Result<()> { ) } -/// Read the single deb822 paragraph of a `.changes`/`.dsc`/`.buildinfo` -/// control file, losslessly, so checksum rewrites preserve the original -/// formatting of untouched fields byte-for-byte. -fn read_control(path: &Path) -> anyhow::Result { - let deb822 = - Deb822::from_file(path).with_context(|| format!("failed to parse {}", path.display()))?; - deb822 - .paragraphs() - .next() - .with_context(|| format!("{} contains no paragraph", path.display())) -} - -fn write_control(paragraph: &Paragraph, path: &Path) -> anyhow::Result<()> { - std::fs::write(path, paragraph.to_string()) - .with_context(|| format!("failed to write {}", path.display())) -} - -/// Filenames ending in `.` listed under `Files:` or any -/// `Checksums-*:` field. -fn child_filename(paragraph: &Paragraph, ext: &str) -> Option { - let suffix = format!(".{ext}"); - for key in paragraph.keys() { - if key != "Files" && !key.starts_with("Checksums-") { - continue; - } - for line in paragraph - .get(&key) - .into_iter() - .flat_map(|v| v.lines().map(str::to_string).collect::>()) - { - if let Some(name) = line.split_whitespace().next_back() - && name.ends_with(&suffix) - { - return Some(name.to_string()); - } - } - } - None -} - -/// The hash algorithms used by the checksum fields of a control file. -#[derive(Clone, Copy)] -enum Hash { - Md5, - Sha1, - Sha256, -} - -impl Hash { - fn hex(self, data: &[u8]) -> String { - fn hex(mut hasher: D, data: &[u8]) -> String { - hasher.update(data); - hasher - .finalize() - .iter() - .map(|b| format!("{b:02x}")) - .collect() - } - match self { - Hash::Md5 => hex(Md5::new(), data), - Hash::Sha1 => hex(Sha1::new(), data), - Hash::Sha256 => hex(Sha256::new(), data), - } - } -} - -/// The checksum fields debmagic understands, mapped to their hash. -const CHECKSUM_FIELDS: &[(&str, Hash)] = &[ - ("Files", Hash::Md5), - ("Checksums-Sha1", Hash::Sha1), - ("Checksums-Sha256", Hash::Sha256), -]; - -/// Rewrite one `Files:`/`Checksums-*:` line: the first token is the -/// checksum, the second the size, the last the filename; entries for -/// other files pass through unchanged. -fn rewrite_checksum_line(line: &str, filename: &str, checksum: &str, size: usize) -> String { - let tokens: Vec<&str> = line.split_whitespace().collect(); - match tokens.as_slice() { - [old_checksum, old_size, middle @ .., name] if *name == filename => { - let middle = if middle.is_empty() { - String::new() - } else { - format!(" {}", middle.join(" ")) - }; - format!("{checksum} {size}{middle} {name}") - } - _ => line.to_string(), - } -} - -/// Rewrite the size and checksum entries for the file listings in a control file -fn fixup_checksums(paragraph: &mut Paragraph, filename: &str, data: &[u8]) -> anyhow::Result<()> { - let size = data.len(); - - for key in paragraph.keys() { - if key.starts_with("Checksums-") && !CHECKSUM_FIELDS.iter().any(|(field, ..)| *field == key) - { - // An unknown checksum format would keep a stale checksum for a - // re-signed file, producing an upload that fails verification - // far away from here. - bail!("unknown checksum field '{key}:' in control file"); - } - } - - for (key, hash) in CHECKSUM_FIELDS { - let Some(value) = paragraph.get(key) else { - continue; - }; - let checksum = hash.hex(data); - let updated = value - .lines() - .filter(|line| !line.is_empty()) - .map(|line| rewrite_checksum_line(line, filename, &checksum, size)) - .collect::>() - .join("\n"); - paragraph.set(key, &updated); - } - Ok(()) -} - /// Is the file already clearsigned? fn is_signed(path: &Path) -> anyhow::Result { let first = std::fs::read_to_string(path) @@ -229,7 +223,7 @@ fn guess_signas(options: &SignOptions, control: &Paragraph) -> String { } /// Substitute the `{file}`, `{key}` and `{email}` placeholders in one -/// `sign.command` argument, rejecting unknown or unterminated ones. +/// `sign.sign_command` argument, rejecting unknown or unterminated ones. fn substitute_placeholders( arg: &str, file: &str, @@ -243,17 +237,19 @@ fn substitute_placeholders( let after = &rest[start + 1..]; let end = after .find('}') - .with_context(|| format!("unterminated '{{' in sign.command argument '{arg}'"))?; + .with_context(|| format!("unterminated '{{' in sign.sign_command argument '{arg}'"))?; let name = &after[..end]; let value = match name { "file" => file, "key" => key, "email" => match email { Some(email) => email, - None => bail!("{{email}} used in sign.command but '{key}' contains no address"), + None => { + bail!("{{email}} used in sign.sign_command but '{key}' contains no address") + } }, _ => bail!( - "unknown placeholder '{{{name}}}' in sign.command (supported: {{file}}, {{key}}, {{email}})" + "unknown placeholder '{{{name}}}' in sign.sign_command (supported: {{file}}, {{key}}, {{email}})" ), }; out.push_str(value); @@ -311,11 +307,11 @@ fn sign_one(path: &Path, signas: &str, options: &SignOptions) -> anyhow::Result< } SignTool::Custom => { let command = options - .command + .sign_command .as_deref() - .with_context(|| "sign.tool = \"custom\" requires sign.command to be set")?; + .with_context(|| "sign.tool = \"custom\" requires sign.sign_command to be set")?; let mut parts = command.split_whitespace(); - let program = parts.next().with_context(|| "sign.command is empty")?; + let program = parts.next().with_context(|| "sign.sign_command is empty")?; cmd = Command::new(program); let file = path.display().to_string(); let email = signer_email(signas); @@ -331,25 +327,15 @@ fn sign_one(path: &Path, signas: &str, options: &SignOptions) -> anyhow::Result< } } if pipes_stdin { - cmd.args(["--output", "-", "-"]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()); - } else { - cmd.stdout(Stdio::piped()); + cmd.args(["--output", "-", "-"]); } - - let mut child = cmd - .spawn() - .with_context(|| format!("failed to run signing command for {}", path.display()))?; + let mut builder = crate::subprocess::command(cmd).capture(Capture::STDOUT); if pipes_stdin { - use std::io::Write; - child - .stdin - .take() - .expect("stdin is piped") - .write_all(&to_sign) - .with_context(|| format!("failed to pipe {} to the signing command", path.display()))?; + builder = builder.input(&to_sign); } + let child = builder + .spawn() + .with_context(|| format!("failed to run the signing command for {}", path.display()))?; let output = child .wait_with_output() .with_context(|| format!("waiting for the signing command of {}", path.display()))?; @@ -485,51 +471,13 @@ mod tests { content.parse().unwrap() } - #[test] - fn child_filename_finds_dsc_and_buildinfo() { - let control = parse_control( - "Format: 1.8\nSource: pkg\nFiles:\n abc 123 pkg_1.0.dsc\n def 456 pkg_1.0.buildinfo\n ghi 789 other.txt\nChecksums-Sha256:\n xyz 123 pkg_1.0.dsc\n", - ); - assert_eq!( - child_filename(&control, "dsc").as_deref(), - Some("pkg_1.0.dsc") - ); - assert_eq!( - child_filename(&control, "buildinfo").as_deref(), - Some("pkg_1.0.buildinfo") - ); - assert_eq!(child_filename(&control, "deb"), None); - } - - #[test] - fn fixup_rewrites_all_checksum_sections() { - let mut control = parse_control( - "Format: 1.8\nFiles:\n oldmd5 3 hash optional pkg_1.0.dsc\nChecksums-Sha1:\n oldsha1 3 pkg_1.0.dsc\nChecksums-Sha256:\n oldsha256 3 pkg_1.0.dsc\n", - ); - let data = b"abc"; - fixup_checksums(&mut control, "pkg_1.0.dsc", data).unwrap(); - - fn hex(mut hasher: D, data: &[u8]) -> String { - hasher.update(data); - hasher - .finalize() - .iter() - .map(|b| format!("{b:02x}")) - .collect() - } - let md5 = hex(Md5::new(), data); - let joined = control.to_string(); - assert!(joined.contains(&format!("{md5} 3 hash optional pkg_1.0.dsc"))); - assert!(joined.contains(&format!(" {} 3 pkg_1.0.dsc", hex(Sha1::new(), data)))); - assert!(joined.contains(&format!(" {} 3 pkg_1.0.dsc", hex(Sha256::new(), data)))); - } - #[test] fn guess_signas_prefers_key_then_changed_by() { let options = SignOptions { key: Some("mykey".into()), tool: SignTool::Gpg, - command: None, + sign_command: None, + verify_command: None, }; let control = parse_control("Maintainer: A \nChanged-By: B \n"); @@ -569,6 +517,81 @@ mod tests { assert_eq!(signer_email("ABC1234"), None); } + #[test] + fn verify_placeholders_substitute() { + assert_eq!( + substitute_verify_placeholders( + "verify --keyring {keyring} {file} {signature}", + Path::new("/tmp/f.tar"), + Path::new("/tmp/f.tar.asc"), + Path::new("/tmp/key.asc"), + ) + .unwrap(), + "verify --keyring /tmp/key.asc /tmp/f.tar /tmp/f.tar.asc" + ); + assert!( + substitute_verify_placeholders( + "{typo}", + Path::new("f"), + Path::new("s"), + Path::new("k") + ) + .is_err() + ); + assert!( + substitute_verify_placeholders( + "{unterminated", + Path::new("f"), + Path::new("s"), + Path::new("k") + ) + .is_err() + ); + } + + #[test] + fn verify_signature_custom_runs_command() { + // a custom command that exits 0 verifies; one that exits 1 fails + let dir = std::env::temp_dir().join("debmagic-sign-test-verify"); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("f.tar"); + std::fs::write(&file, "x").unwrap(); + let sig = dir.join("f.tar.asc"); + std::fs::write(&sig, "x").unwrap(); + let keyring = dir.join("key.asc"); + std::fs::write(&keyring, "x").unwrap(); + + let ok = SignOptions { + verify_command: Some("true {signature}".to_string()), + tool: SignTool::Custom, + ..Default::default() + }; + assert!(verify_signature(&ok, &file, &sig, &keyring).is_ok()); + + let failing = SignOptions { + verify_command: Some("false {signature}".to_string()), + tool: SignTool::Custom, + ..Default::default() + }; + assert!(verify_signature(&failing, &file, &sig, &keyring).is_err()); + + // verify_command falls back to sign_command + let fallback = SignOptions { + sign_command: Some("true {signature}".to_string()), + tool: SignTool::Custom, + ..Default::default() + }; + assert!(verify_signature(&fallback, &file, &sig, &keyring).is_ok()); + + // custom without any command configured is an error, not a gpg fallback + let missing = SignOptions { + tool: SignTool::Custom, + ..Default::default() + }; + assert!(verify_signature(&missing, &file, &sig, &keyring).is_err()); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn unsign_strips_armor() { let dir = std::env::temp_dir().join("debmagic-sign-test-unsign"); diff --git a/packages/debmagic/src/subprocess.rs b/packages/debmagic/src/subprocess.rs new file mode 100644 index 00000000..710e7f56 --- /dev/null +++ b/packages/debmagic/src/subprocess.rs @@ -0,0 +1,233 @@ +use std::io::Write; +use std::process::{Child, Command, Stdio}; + +/// Which output streams of a command run are captured into its +/// [`CommandResult`] instead of passing through to the user's terminal. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Capture { + pub stdout: bool, + pub stderr: bool, +} + +impl Capture { + pub const NONE: Self = Self { + stdout: false, + stderr: false, + }; + pub const STDOUT: Self = Self { + stdout: true, + stderr: false, + }; + pub const STDERR: Self = Self { + stdout: false, + stderr: true, + }; + pub const ALL: Self = Self { + stdout: true, + stderr: true, + }; +} + +/// Result of one command run, like Python's `CompletedProcess`: the exit +/// code, plus each stream the caller asked to capture via [`Capture`]. +#[derive(Debug, Clone)] +pub struct CommandResult { + pub exit_code: i32, + /// Captured stdout; `None` unless the caller captured it. + pub stdout: Option, + /// Captured stderr; `None` unless the caller captured it. + pub stderr: Option, +} + +/// Pipe the streams selected by `capture`; the rest stay inherited. +fn pipe_streams(command: &mut Command, capture: Capture) { + if capture.stdout { + command.stdout(Stdio::piped()); + } + if capture.stderr { + command.stderr(Stdio::piped()); + } +} + +/// Collect the piped streams of a finished child into a [`CommandResult`]. +fn finish(child: Child, capture: Capture) -> std::io::Result { + let output = child.wait_with_output()?; + Ok(CommandResult { + exit_code: output.status.code().unwrap_or(-1), + stdout: capture + .stdout + .then(|| String::from_utf8_lossy(&output.stdout).into_owned()), + stderr: capture + .stderr + .then(|| String::from_utf8_lossy(&output.stderr).into_owned()), + }) +} + +/// Start building a subprocess run of `command`: pick the streams to +/// [`capture`](CommandBuilder::capture), optionally provide +/// [`input`](CommandBuilder::input), then [`run`](CommandBuilder::run) it +/// to completion or [`spawn`](CommandBuilder::spawn) it and handle the +/// child yourself. +/// +/// This is the opt-in capture API: `Command::output()` pipes *every* +/// stream left at its default, silently swallowing uncaptured output, +/// while here uncaptured streams stay inherited and pass through to +/// the user's terminal. +pub fn command(command: Command) -> CommandBuilder<'static> { + CommandBuilder { + command, + input: None, + capture: Capture::NONE, + } +} + +/// A [`Command`] configured through [`command`], waiting to be run. +pub struct CommandBuilder<'a> { + command: Command, + input: Option<&'a [u8]>, + capture: Capture, +} + +impl CommandBuilder<'_> { + /// Write `input` to the command's piped stdin; without it the + /// parent's stdin is inherited. + /// + /// The whole input is written before the child is returned, so a + /// command that answers on stdout before consuming all of stdin can + /// deadlock once its output outgrows the pipe buffer. + pub fn input<'b>(self, input: &'b [u8]) -> CommandBuilder<'b> { + CommandBuilder { + input: Some(input), + ..self + } + } + + /// Capture the selected streams into the [`CommandResult`] instead + /// of passing them through to the user's terminal. + pub fn capture(mut self, capture: Capture) -> Self { + self.capture = capture; + self + } + + /// Spawn the command and wait for it, collecting the captured + /// streams. + pub fn run(self) -> std::io::Result { + let Self { + command, + input, + capture, + } = self; + finish(spawn(command, input, capture)?, capture) + } + + /// Spawn the command and return the running child, so the caller + /// can stream its output while it runs. + pub fn spawn(self) -> std::io::Result { + let Self { + command, + input, + capture, + } = self; + spawn(command, input, capture) + } +} + +fn spawn(mut command: Command, input: Option<&[u8]>, capture: Capture) -> std::io::Result { + if input.is_some() { + command.stdin(Stdio::piped()); + } + pipe_streams(&mut command, capture); + let mut child = command.spawn()?; + if let Some(input) = input { + child + .stdin + .take() + .expect("stdin is piped") + .write_all(input)?; + } + Ok(child) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn uncaptured_streams_pass_through() { + let result = command(Command::new("true")).run().unwrap(); + assert_eq!(result.exit_code, 0); + assert_eq!(result.stdout, None); + assert_eq!(result.stderr, None); + } + + #[test] + fn input_is_written_to_stdin() { + let result = command(Command::new("cat")) + .input(b"piped") + .capture(Capture::STDOUT) + .run() + .unwrap(); + assert_eq!(result.exit_code, 0); + assert_eq!(result.stdout.as_deref(), Some("piped")); + } + + #[test] + fn stdout_capture_collects_only_stdout() { + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo out; echo err >&2"]); + let result = command(cmd).capture(Capture::STDOUT).run().unwrap(); + assert_eq!(result.exit_code, 0); + assert_eq!(result.stdout.as_deref(), Some("out\n")); + assert_eq!(result.stderr, None); + } + + #[test] + fn stderr_capture_collects_only_stderr() { + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo out; echo err >&2"]); + let result = command(cmd).capture(Capture::STDERR).run().unwrap(); + assert_eq!(result.exit_code, 0); + assert_eq!(result.stdout, None); + assert_eq!(result.stderr.as_deref(), Some("err\n")); + } + + #[test] + fn all_capture_collects_both_streams() { + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo out; echo err >&2"]); + let result = command(cmd).capture(Capture::ALL).run().unwrap(); + assert_eq!(result.exit_code, 0); + assert_eq!(result.stdout.as_deref(), Some("out\n")); + assert_eq!(result.stderr.as_deref(), Some("err\n")); + } + + #[test] + fn exit_code_is_reported_regardless_of_capture() { + let mut cmd = Command::new("sh"); + cmd.args(["-c", "exit 3"]); + let result = command(cmd).capture(Capture::STDOUT).run().unwrap(); + assert_eq!(result.exit_code, 3); + } + + #[test] + fn spawn_without_input_leaves_stdin_inherited() { + // `true`, not `cat`: inherited stdin is the test runner's, which + // never reaches EOF, so a reading child would hang forever + let mut child = command(Command::new("true")).spawn().unwrap(); + assert!(child.stdin.is_none()); + assert!(child.stdout.is_none()); + assert!(child.stderr.is_none()); + assert!(child.wait().unwrap().success()); + } + + #[test] + fn spawn_with_input_leaves_uncaptured_streams_inherited() { + let mut child = command(Command::new("cat")) + .input(b"x") + .spawn() + .unwrap(); + assert!(child.stdout.is_none()); + assert!(child.stderr.is_none()); + assert!(child.wait().unwrap().success()); + } +} diff --git a/packages/debmagic/src/test/intent.rs b/packages/debmagic/src/test/intent.rs index 7c29bf2e..fca4bc1e 100644 --- a/packages/debmagic/src/test/intent.rs +++ b/packages/debmagic/src/test/intent.rs @@ -101,6 +101,7 @@ mod tests { driver_overrides: DriverOverrides { apt_mirror: None, proposed: None, + apt_update_age: None, docker: DriverDockerConfigOverrides { base_image: None }, bare: DriverBareConfigOverrides {}, lxd: DriverLxdConfigOverrides { diff --git a/packages/debmagic/src/test/run.rs b/packages/debmagic/src/test/run.rs index 4ec97e81..0d996c7b 100644 --- a/packages/debmagic/src/test/run.rs +++ b/packages/debmagic/src/test/run.rs @@ -5,6 +5,7 @@ use std::{ }; use super::intent::TestIntent; +use crate::build::artifacts::{copy_changes_artifacts, copy_dir_all, find_changes_file}; use crate::build::source::stage_source_tree; use crate::driver::{ Driver, DriverType, Environment, EnvironmentDriver, EnvironmentMetadata, EnvironmentPurpose, @@ -12,13 +13,11 @@ use crate::driver::{ config::{DriverConfig, DriverOverrides}, create_driver, remove_environment_root, }; -use crate::package::PackageIdentity; -use crate::{ - build::artifacts::{copy_changes_artifacts, copy_dir_all, find_changes_file}, - package::load_package_identity, -}; +use crate::package::load_package; +use crate::subprocess::Capture; use anyhow::{Context, anyhow, bail}; use debmagic_common::distro::DistroVersion; +use debmagic_common::package::SourcePackage; /// autopkgtest(1) exit status values (Debian autopkgtest 6.x). /// Some codes combine categories (e.g. 6 = 4|2); treat them as bitmasks where noted. @@ -44,9 +43,9 @@ struct TestRun { fn get_build_root_and_identifier( temp_build_dir: &Path, - identity: &PackageIdentity, + package: &SourcePackage, ) -> (String, PathBuf) { - let package_identifier = format!("{}-{}", identity.name, identity.version); + let package_identifier = format!("{}-{}", package.name(), package.version()); let build_root = temp_build_dir.join(&package_identifier); (package_identifier, build_root) } @@ -134,7 +133,7 @@ impl TestRun { fn prepare_test_env( intent: &TestIntent, environment: &Environment, - identity: &PackageIdentity, + package: &SourcePackage, changes_path: &Path, ) -> anyhow::Result { let test_root = &environment.root_dir; @@ -150,7 +149,7 @@ fn prepare_test_env( environment .create_dirs() .context("failed to create test directories")?; - stage_source_tree(environment, identity, intent.config.source_sync_mode, false)?; + stage_source_tree(environment, package, intent.config.source_sync_mode, false)?; copy_changes_artifacts(changes_path, &environment.work_dir())?; return Ok(test_run); } @@ -160,7 +159,7 @@ fn prepare_test_env( environment .create_dirs() .context("failed to create test directories")?; - stage_source_tree(environment, identity, intent.config.source_sync_mode, false)?; + stage_source_tree(environment, package, intent.config.source_sync_mode, false)?; copy_changes_artifacts(changes_path, &environment.work_dir())?; let test_run = TestRun::create(environment, &intent.config.driver, &intent.driver_overrides)?; @@ -216,9 +215,9 @@ fn autopkgtest_isolation_args(isolation: IsolationCapability) -> Vec<&'static st } pub fn run_test(intent: &TestIntent) -> anyhow::Result { - let identity = load_package_identity(&intent.source_dir)?; + let package = load_package(&intent.source_dir)?; let (package_identifier, build_root) = - get_build_root_and_identifier(&intent.config.temp_build_dir, &identity); + get_build_root_and_identifier(&intent.config.temp_build_dir, &package); let changes_path = if let Some(ref explicit) = intent.changes { if !explicit.is_file() { @@ -272,7 +271,7 @@ pub fn run_test(intent: &TestIntent) -> anyhow::Result { let environment = Environment { driver, - package_name: identity.name.clone(), + package_name: package.name().to_string(), package_identifier, root_dir: test_root.clone(), distro, @@ -280,7 +279,7 @@ pub fn run_test(intent: &TestIntent) -> anyhow::Result { purpose: EnvironmentPurpose::Test, }; - let test_run = prepare_test_env(intent, &environment, &identity, &changes_path) + let test_run = prepare_test_env(intent, &environment, &package, &changes_path) .context("failed to prepare test environment")?; test_run .write_metadata() @@ -327,10 +326,11 @@ pub fn run_test(intent: &TestIntent) -> anyhow::Result { "null", ]); - crate::output::stage(&format!("Running autopkgtest for {}", identity.name)); + crate::output::stage(&format!("Running autopkgtest for {}", package.name())); let exit_code = test_run .driver - .run_command(&autopkgtest_cmd, &work_dir, true, &[]) + .run_command(&autopkgtest_cmd, &work_dir, true, &[], Capture::NONE) + .map(|result| result.exit_code) .unwrap_or(-1); let summary_path = autopkgtest_out_host.join("summary"); @@ -392,21 +392,19 @@ pub fn run_test(intent: &TestIntent) -> anyhow::Result { #[cfg(test)] mod tests { use super::*; - use crate::package::PackageIdentity; - use debmagic_common::debian::version::PackageVersion; - - fn sample_identity() -> PackageIdentity { - PackageIdentity { - name: "pkg".to_string(), - version: PackageVersion::new(None, "1.0".to_string(), Some("1".to_string())), - source_dir: PathBuf::from("/src"), - } + + fn sample_package() -> SourcePackage { + SourcePackage::from_files([( + "changelog", + "pkg (1.0-1) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n", + )]) + .unwrap() } #[test] fn test_build_root_appends_test_suffix() { let (_, build_root) = - get_build_root_and_identifier(Path::new("/tmp/debmagic"), &sample_identity()); + get_build_root_and_identifier(Path::new("/tmp/debmagic"), &sample_package()); assert_eq!(build_root, PathBuf::from("/tmp/debmagic/pkg-1.0-1")); assert_eq!( test_build_root(&build_root), diff --git a/packages/debmagic/src/time.rs b/packages/debmagic/src/time.rs new file mode 100644 index 00000000..8e04d15c --- /dev/null +++ b/packages/debmagic/src/time.rs @@ -0,0 +1,179 @@ +use std::time::{Duration, SystemTime}; + +use anyhow::anyhow; +use chrono::{DateTime, SecondsFormat, Utc}; +use serde::{Deserialize, Serialize}; + +/// When something cached is refreshed again: before every use, only the first +/// time, or once its last refresh is older than a maximum age. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub enum RefreshPolicy { + /// Refresh before every use. + Now, + /// Only refresh the first time. + Never, + /// Refresh again once the last one is older than this. + OlderThan(Duration), +} + +impl Default for RefreshPolicy { + fn default() -> Self { + Self::OlderThan(Duration::from_secs(24 * 60 * 60)) + } +} + +impl RefreshPolicy { + fn parse(value: &str) -> anyhow::Result { + if value == "now" { + return Ok(Self::Now); + } + if value == "never" { + return Ok(Self::Never); + } + + let invalid = || { + anyhow!( + "invalid refresh policy '{value}': expected 'now', 'never' or a duration like '1d', '12h', '30m'" + ) + }; + let Some((digits, unit)) = value.split_at_checked(value.len().saturating_sub(1)) else { + return Err(invalid()); + }; + let amount: u64 = digits.parse().map_err(|_| invalid())?; + let seconds = match unit { + "s" => Some(amount), + "m" => amount.checked_mul(60), + "h" => amount.checked_mul(60 * 60), + "d" => amount.checked_mul(24 * 60 * 60), + _ => return Err(invalid()), + } + .ok_or_else(|| anyhow!("refresh policy '{value}' is too large"))?; + Ok(Self::OlderThan(Duration::from_secs(seconds))) + } +} + +impl std::fmt::Display for RefreshPolicy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match *self { + Self::Now => f.write_str("now"), + Self::Never => f.write_str("never"), + Self::OlderThan(age) => { + let seconds = age.as_secs(); + let (amount, unit) = if seconds.is_multiple_of(24 * 60 * 60) { + (seconds / (24 * 60 * 60), "d") + } else if seconds.is_multiple_of(60 * 60) { + (seconds / (60 * 60), "h") + } else if seconds.is_multiple_of(60) { + (seconds / 60, "m") + } else { + (seconds, "s") + }; + write!(f, "{amount}{unit}") + } + } + } +} + +impl From for String { + fn from(policy: RefreshPolicy) -> Self { + policy.to_string() + } +} + +impl TryFrom for RefreshPolicy { + type Error = anyhow::Error; + + fn try_from(value: String) -> anyhow::Result { + Self::parse(&value) + } +} + +impl std::str::FromStr for RefreshPolicy { + type Err = anyhow::Error; + + fn from_str(value: &str) -> anyhow::Result { + Self::parse(value) + } +} + +/// UTC timestamp in RFC 3339 with second precision +pub fn iso_timestamp() -> String { + Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true) +} + +/// Parse a timestamp written by [`iso_timestamp`] +pub fn parse_iso_timestamp(value: &str) -> Option { + DateTime::parse_from_rfc3339(value.trim()) + .ok() + .map(|time| time.into()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_now_and_never() { + assert_eq!(RefreshPolicy::parse("now").unwrap(), RefreshPolicy::Now); + assert_eq!(RefreshPolicy::parse("never").unwrap(), RefreshPolicy::Never); + } + + #[test] + fn parses_durations() { + let age = |seconds| RefreshPolicy::OlderThan(Duration::from_secs(seconds)); + assert_eq!(RefreshPolicy::parse("45s").unwrap(), age(45)); + assert_eq!(RefreshPolicy::parse("30m").unwrap(), age(30 * 60)); + assert_eq!(RefreshPolicy::parse("12h").unwrap(), age(12 * 60 * 60)); + assert_eq!(RefreshPolicy::parse("1d").unwrap(), age(24 * 60 * 60)); + } + + #[test] + fn rejects_invalid_values() { + for value in ["", "1", "d", "1x", "now!", "1.5d"] { + assert!( + RefreshPolicy::parse(value).is_err(), + "{value} should not parse" + ); + } + } + + #[test] + fn defaults_to_one_day() { + assert_eq!( + RefreshPolicy::default(), + RefreshPolicy::OlderThan(Duration::from_secs(24 * 60 * 60)) + ); + } + + #[test] + fn displays_roundtrippably() { + assert_eq!(RefreshPolicy::Now.to_string(), "now"); + assert_eq!(RefreshPolicy::Never.to_string(), "never"); + assert_eq!( + RefreshPolicy::OlderThan(Duration::from_secs(24 * 60 * 60)).to_string(), + "1d" + ); + assert_eq!( + RefreshPolicy::OlderThan(Duration::from_secs(90 * 60)).to_string(), + "90m" + ); + assert_eq!( + RefreshPolicy::OlderThan(Duration::from_secs(37)).to_string(), + "37s" + ); + } + + #[test] + fn iso_timestamps_roundtrip() { + let parsed = parse_iso_timestamp(&iso_timestamp()).unwrap(); + let age = SystemTime::now().duration_since(parsed).unwrap(); + assert!(age < Duration::from_secs(120)); + } + + #[test] + fn iso_timestamp_parsing_rejects_garbage() { + assert_eq!(parse_iso_timestamp("not a timestamp"), None); + assert_eq!(parse_iso_timestamp(""), None); + } +} diff --git a/packages/debmagic/src/upload.rs b/packages/debmagic/src/upload.rs new file mode 100644 index 00000000..e634f02d --- /dev/null +++ b/packages/debmagic/src/upload.rs @@ -0,0 +1,253 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Context, bail}; + +use crate::upload::{scp::ScpUploader, sftp::SftpUploader, target::substitute_placeholders}; + +pub mod orig; +pub mod scp; +pub mod sftp; +pub mod ssh; +pub mod target; + +pub use target::{ + UploadConfig, UploadOverrides, UploadTarget, UploadTargetConfig, builtin_targets, + parse_target_spec, resolve_target, +}; + +/// One upload method implementation. A new method (e.g. git-based) +/// is a new `UploadMethod` variant plus an implementation of this trait. +pub trait Uploader { + /// Upload all `files` to the target's incoming dir, keeping each + /// one's file name. Implementations should batch the files into a + /// single session where the transport allows it. + fn upload_files(&mut self, files: &[PathBuf]) -> anyhow::Result<()>; +} + +/// The upload methods debmagic supports. +#[derive( + Debug, + Default, + Copy, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + clap::ValueEnum, +)] +#[serde(rename_all = "snake_case")] +pub enum UploadMethod { + /// `scp :/` + #[default] + Scp, + /// `sftp -b` batch mode, `put`ting each file into `incoming`. + Sftp, +} + +/// Create the uploader for a target's configured method. +pub fn create_uploader(target: &UploadTarget) -> anyhow::Result> { + let method = target.config.method.unwrap_or_default(); + match method { + UploadMethod::Scp => Ok(Box::new(ScpUploader { + target: target.clone(), + })), + UploadMethod::Sftp => Ok(Box::new(SftpUploader { + target: target.clone(), + })), + } +} + +/// Read the local file names listed in a `.changes` file, plus +/// the `.changes` itself: everything that must be uploaded. +pub fn changes_upload_files(changes_path: &Path) -> anyhow::Result> { + let changes = crate::control::read_changes(changes_path)?; + let files = changes + .files() + .with_context(|| format!("{} has no Files field", changes_path.display()))?; + let dir = changes_path + .parent() + .context("changes file has no parent directory")?; + let mut paths: Vec = Vec::with_capacity(files.len() + 1); + for file in files { + paths.push(dir.join(file.filename.clone())); + } + paths.push(changes_path.to_path_buf()); + Ok(paths) +} + +/// Run a target's `pre_upload_commands`, each via `sh -c`, with the +/// `{changes}` placeholder substituted and `DEBMAGIC_UPLOAD_*` env vars +/// set. A non-zero exit aborts the upload before anything is transferred. +pub fn run_pre_upload_commands( + target: &UploadTarget, + changes_file: &Path, + commands: &[String], +) -> anyhow::Result<()> { + for command in commands { + let command = + substitute_placeholders(command, &[("changes", &changes_file.to_string_lossy())]); + println!("debmagic: running pre-upload command: {command}"); + let status = std::process::Command::new("sh") + .arg("-c") + .arg(&command) + .env("DEBMAGIC_UPLOAD_CHANGES", changes_file) + .env("DEBMAGIC_UPLOAD_TARGET", &target.name) + .env("DEBMAGIC_UPLOAD_TARGET_SERVER", &target.config.server) + .env("DEBMAGIC_UPLOAD_TARGET_INCOMING", target.incoming_dir()) + .status() + .with_context(|| format!("failed to run pre-upload command: {command}"))?; + if !status.success() { + bail!( + "pre-upload command failed (exit status: {status}), aborting the upload: {command}" + ); + } + } + Ok(()) +} + +/// Upload a `.changes` file and everything it references to a target: +/// run the pre-upload checks, then upload the referenced files and +/// the `.changes` itself. A successful upload is recorded in the +/// `.upload.json` next to the `.changes`; a prior successful upload +/// to the same target is refused unless `force` is set. +pub fn upload_changes( + target: &UploadTarget, + target_spec: &str, + changes_file: &Path, + skip_hooks: bool, + force: bool, + upstream_version: Option<&str>, +) -> anyhow::Result<()> { + let mut log = UploadLog::load(changes_file)?; + if !force && log.contains_target(target_spec) { + bail!( + "{target_spec} already has a successful upload of {} recorded in {}; pass --force to upload anyway", + changes_file.display(), + UploadLog::path_for(changes_file)?.display() + ); + } + + let files = changes_upload_files(changes_file)?; + + if !skip_hooks { + run_pre_upload_commands(target, changes_file, &target.config.pre_upload_commands) + .context("running the pre-upload commands failed")?; + } + + let mut uploader = create_uploader(target)?; + uploader.upload_files(&files).with_context(|| { + format!( + "transferring {} files to {} failed", + files.len(), + target.name + ) + })?; + log.record(changes_file, target_spec, upstream_version)?; + Ok(()) +} + +/// One completed upload, as recorded in the `.upload.json` file +/// next to the `.changes`. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct UploadRecord { + /// The full target spec as invoked, e.g. `ppa:user/repo`. + pub target: String, + /// Local time of the completed upload, ISO 8601 with UTC offset + /// (like python's `datetime.isoformat()`). + pub time: String, + /// The upstream version of the uploaded source + #[serde(default, skip_serializing_if = "Option::is_none")] + pub upstream_version: Option, +} + +/// The `.upload.json` file next to a `.changes`, holding the record of +/// every successful upload of it. +#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize)] +#[serde(default)] +pub struct UploadLog { + pub uploads: Vec, +} + +impl UploadLog { + /// Path of the log for a `.changes` file: + /// `.upload.json` next to it. + pub fn path_for(changes_file: &Path) -> anyhow::Result { + let stem = changes_file + .file_stem() + .context("changes file has no file stem")?; + let dir = changes_file + .parent() + .context("changes file has no parent directory")?; + Ok(dir.join(format!("{}.upload.json", stem.to_string_lossy()))) + } + + pub fn load(changes_file: &Path) -> anyhow::Result { + let path = Self::path_for(changes_file)?; + if !path.is_file() { + return Ok(Self::default()); + } + let content = std::fs::read_to_string(&path) + .with_context(|| format!("failed to read {}", path.display()))?; + serde_json::from_str(&content) + .with_context(|| format!("failed to parse {}", path.display())) + } + + /// Was this exact target spec already uploaded successfully? + pub fn contains_target(&self, target_spec: &str) -> bool { + self.uploads.iter().any(|r| r.target == target_spec) + } + + /// Append a record for a successful upload and write the file back. + pub fn record( + &mut self, + changes_file: &Path, + target_spec: &str, + upstream_version: Option<&str>, + ) -> anyhow::Result<()> { + self.uploads.push(UploadRecord { + target: target_spec.to_string(), + time: chrono::Local::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, false), + upstream_version: upstream_version.map(str::to_string), + }); + let path = Self::path_for(changes_file)?; + let content = serde_json::to_string_pretty(self).context("serializing upload log")?; + std::fs::write(&path, content) + .with_context(|| format!("failed to write {}", path.display())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_upload_log_roundtrip_and_check() { + let dir = std::env::temp_dir().join(format!("debmagic-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let changes = dir.join("pkg_1.0-1_source.changes"); + std::fs::write(&changes, "Format: 1.8\n").unwrap(); + + let mut log = UploadLog::load(&changes).unwrap(); + assert!(!log.contains_target("ppa:sfttech/debmagic")); + log.record(&changes, "ppa:sfttech/debmagic", Some("1.0")) + .unwrap(); + assert!(log.contains_target("ppa:sfttech/debmagic")); + // a different ppa is not a duplicate + assert!(!log.contains_target("ppa:other/ppa")); + + // persists across loads + let reloaded = UploadLog::load(&changes).unwrap(); + assert!(reloaded.contains_target("ppa:sfttech/debmagic")); + assert_eq!(reloaded.uploads.len(), 1); + // python isoformat-compatible, e.g. "2026-09-18T14:23:01+02:00" + let time = &reloaded.uploads[0].time; + assert_eq!(time.len(), 25); + assert_eq!(&time[4..5], "-"); + assert_eq!(&time[10..11], "T"); + assert_eq!(&time[13..14], ":"); + assert!(time[19..].starts_with('+') || time[19..].starts_with('-')); + + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/packages/debmagic/src/upload/orig.rs b/packages/debmagic/src/upload/orig.rs new file mode 100644 index 00000000..0e474443 --- /dev/null +++ b/packages/debmagic/src/upload/orig.rs @@ -0,0 +1,316 @@ +use std::path::Path; + +use anyhow::Context; + +use crate::sign::SignOptions; +use debmagic_common::package::SourcePackage; + +/// Whether an upload should include the `orig` tarball. +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, clap::ValueEnum)] +pub enum IncludeOrig { + /// Include only when the archive cannot have this upstream + /// version's orig yet: the changelog head bumps the upstream part + /// relative to the entry below, or is a deltarebase onto a Debian + /// version Ubuntu never had. Everything else — the usual + /// Ubuntu-revision-on-top upload — the archive already carries + /// the orig. + #[default] + Auto, + Yes, + No, +} + +/// Resolve `--include-orig` against the changelog: `auto` includes +/// the orig only when the archive provably lacks it — a new upstream +/// version, or a deltarebase onto a Debian version Ubuntu never had. +/// For every other upload the archive already has it, so the default +/// is to not send it again. +pub fn decide_orig_upload(mode: IncludeOrig, source_dir: &Path) -> anyhow::Result { + match mode { + IncludeOrig::Yes => Ok(true), + IncludeOrig::No => Ok(false), + IncludeOrig::Auto => { + let changelog = crate::changelog::load_changelog(source_dir)?; + Ok( + debmagic_common::debian::changelog::is_new_upstream_version(&changelog) + || debmagic_common::debian::changelog::is_deltarebase_onto_debian(&changelog), + ) + } + } +} + +/// Apply an orig-inclusion decision to a `.changes` file: add or +/// remove the orig tarball entries (main and components), re-signing +/// with `sign_options` when the file changes. +pub fn changes_include_orig( + changes_file: &Path, + include: bool, + package: &SourcePackage, + sign_options: &SignOptions, +) -> anyhow::Result<()> { + if package.is_native() { + return Ok(()); + } + let orig_names: Vec = if include { + let dir = changes_file + .parent() + .context("changes file has no parent directory")?; + let mut names = Vec::new(); + if let Some(tarball) = debmagic_common::changes::find_orig_in_dir( + dir, + package.name(), + package.version().upstream_version(), + ) { + names.push(tarball); + } + // component origs: any orig- tarball next to the .changes + for entry in dir + .read_dir() + .with_context(|| format!("failed to read {}", dir.display()))? + .flatten() + { + let name = entry.file_name().to_string_lossy().into_owned(); + if debmagic_common::changes::is_orig_tarball(&name) + && name.contains(".orig-") + && name.starts_with(&format!( + "{}_{}.", + package.name(), + package.version().upstream_version() + )) + { + names.push(entry.path()); + } + } + if names.is_empty() { + anyhow::bail!( + "no orig tarball for {} {} is next to {}", + package.name(), + package.version().upstream_version(), + changes_file.display() + ); + } + for tarball in &names { + verify_against_dsc(tarball, changes_file)?; + } + names + .into_iter() + .filter_map(|p| p.file_name().map(|n| n.to_string_lossy().into_owned())) + .collect() + } else { + Vec::new() + }; + + let mut modified = if !include { + crate::changes::set_changes_orig(changes_file, None)? + } else { + false + }; + for name in &orig_names { + modified |= crate::changes::set_changes_orig(changes_file, Some(name))?; + } + if modified { + println!( + "debmagic: re-signing {} after the orig tarball change", + changes_file.display() + ); + crate::sign::sign_file(changes_file, sign_options, false, package.name())?; + } + Ok(()) +} + +/// Verify the orig tarball against the checksums the `.dsc` referenced by +/// the `.changes` recorded for it — a mismatch means the tarball next to +/// the `.changes` is stale. Every checksum the `.dsc` records for the +/// tarball is checked; a missing `.dsc` or one listing no entry for the +/// tarball is skipped. +fn verify_against_dsc(tarball: &Path, changes_file: &Path) -> anyhow::Result<()> { + let Some(dir) = changes_file.parent() else { + return Ok(()); + }; + let changes = crate::control::read_changes(changes_file)?; + let Some(files) = changes.files() else { + return Ok(()); + }; + let Some(dsc_name) = files + .iter() + .find(|f| f.filename.ends_with(".dsc")) + .map(|f| f.filename.clone()) + else { + return Ok(()); + }; + let dsc_path = dir.join(&dsc_name); + let dsc = crate::control::read_control(&dsc_path) + .with_context(|| format!("failed to read {dsc_name}"))?; + let tarball_name = tarball + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + + let digests = crate::control::digest_file(tarball)?; + + debmagic_common::debian::control::verify_checksums(&dsc, &tarball_name, &digests).map_err( + |error| { + anyhow::anyhow!( + "{error}; the tarball next to the .changes is stale (recorded by {dsc_name})" + ) + }, + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sha256_hex(data: &[u8]) -> String { + use sha2::Digest; + let mut hasher = sha2::Sha256::new(); + hasher.update(data); + hasher + .finalize() + .iter() + .map(|b| format!("{b:02x}")) + .collect() + } + + #[test] + fn test_verify_against_dsc() { + let dir = std::env::temp_dir().join(format!("debmagic-dsc-verify-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let tarball = dir.join("pkg_1.0.orig.tar.xz"); + std::fs::write(&tarball, "tarball content").unwrap(); + let changes = dir.join("pkg_1.0-1_source.changes"); + std::fs::write( + &changes, + format!( + "Format: 1.8\nFiles:\n {} 3 debiandev optional pkg_1.0-1.dsc\n", + sha256_hex(b"dsc") + ), + ) + .unwrap(); + let dsc = dir.join("pkg_1.0-1.dsc"); + std::fs::write( + &dsc, + format!( + "Format: 3.0 (quilt)\nChecksums-Sha256:\n {} 15 pkg_1.0.orig.tar.xz\n", + sha256_hex(b"tarball content") + ), + ) + .unwrap(); + + // matching checksum passes + verify_against_dsc(&tarball, &changes).unwrap(); + + // stale tarball fails + std::fs::write(&tarball, "different content").unwrap(); + assert!(verify_against_dsc(&tarball, &changes).is_err()); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn test_resolve_include_orig_modes() { + // Yes/No never consult any state; auto is covered by + // test_resolve_include_orig_auto below + let base = std::env::temp_dir().join(format!("debmagic-include-{}", std::process::id())); + let source = base.join("pkg"); + std::fs::create_dir_all(source.join("debian")).unwrap(); + std::fs::write( + source.join("debian").join("changelog"), + "pkg (1.0-1) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n", + ) + .unwrap(); + + assert!(decide_orig_upload(IncludeOrig::Yes, &source).unwrap()); + assert!(!decide_orig_upload(IncludeOrig::No, &source).unwrap()); + + std::fs::remove_dir_all(&base).unwrap(); + } + + #[test] + fn test_resolve_include_orig_auto() { + // auto: a plain new Ubuntu revision — the archive already has + // the orig, so it must not ride along + let base = + std::env::temp_dir().join(format!("debmagic-include-auto-{}", std::process::id())); + let source = base.join("pkg"); + std::fs::create_dir_all(source.join("debian")).unwrap(); + std::fs::write( + source.join("debian").join("changelog"), + "pkg (1.0-1ubuntu2) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n\n" + .to_string() + + "pkg (1.0-1ubuntu1) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n", + ) + .unwrap(); + assert!(!decide_orig_upload(IncludeOrig::Auto, &source).unwrap()); + + // a deltarebase onto a Debian version: the archive cannot + // have the orig yet, so it rides along + std::fs::write( + source.join("debian").join("changelog"), + "pkg (1.0-1ubuntu1) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n\n" + .to_string() + + "pkg (1.0-1) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n", + ) + .unwrap(); + assert!(decide_orig_upload(IncludeOrig::Auto, &source).unwrap()); + + // a new upstream version: the archive cannot have its orig + // yet either, so it rides along too + std::fs::write( + source.join("debian").join("changelog"), + "pkg (1.1.0-1ubuntu1) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n\n" + .to_string() + + "pkg (1.0.0-1ubuntu3) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n", + ) + .unwrap(); + assert!(decide_orig_upload(IncludeOrig::Auto, &source).unwrap()); + + std::fs::remove_dir_all(&base).unwrap(); + } + + #[test] + fn test_changes_include_orig_removes_entries() { + let base = + std::env::temp_dir().join(format!("debmagic-include-remove-{}", std::process::id())); + let source = base.join("pkg"); + std::fs::create_dir_all(source.join("debian")).unwrap(); + std::fs::write( + source.join("debian").join("changelog"), + "pkg (1.0-1) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n", + ) + .unwrap(); + let changes = base.join("pkg_1.0-1_source.changes"); + std::fs::write( + &changes, + "Format: 1.8\nMaintainer: A \nFiles:\n abc 3 debiandev optional pkg_1.0-1.dsc\nChecksums-Sha256:\n xyz 3 pkg_1.0-1.dsc\n", + ) + .unwrap(); + std::fs::write(base.join("pkg_1.0.orig.tar.xz"), "tarball").unwrap(); + std::fs::write(base.join("pkg_1.0-1.dsc"), "Format: 3.0 (quilt)\n").unwrap(); + + let package = crate::package::load_package(&source).unwrap(); + + // a custom "signing" command that just cats the file back keeps + // the test independent of a gpg keyring on the host + let sign_options = SignOptions { + tool: crate::sign::SignTool::Custom, + sign_command: Some("cat".to_string()), + ..Default::default() + }; + + // add: the orig entry appears + changes_include_orig(&changes, true, &package, &sign_options).unwrap(); + let content = std::fs::read_to_string(&changes).unwrap(); + assert!(content.contains("pkg_1.0.orig.tar.xz")); + + // remove: the orig entry is gone again + changes_include_orig(&changes, false, &package, &sign_options).unwrap(); + let content = std::fs::read_to_string(&changes).unwrap(); + assert!(!content.contains("orig.tar")); + // the non-orig entries survive + assert!(content.contains("pkg_1.0-1.dsc")); + + std::fs::remove_dir_all(&base).unwrap(); + } +} diff --git a/packages/debmagic/src/upload/scp.rs b/packages/debmagic/src/upload/scp.rs new file mode 100644 index 00000000..4bc90913 --- /dev/null +++ b/packages/debmagic/src/upload/scp.rs @@ -0,0 +1,36 @@ +use std::path::PathBuf; + +use anyhow::Context; + +use super::{UploadTarget, Uploader}; + +pub struct ScpUploader { + pub target: UploadTarget, +} + +impl Uploader for ScpUploader { + fn upload_files(&mut self, files: &[PathBuf]) -> anyhow::Result<()> { + if files.is_empty() { + return Ok(()); + } + let host = self.target.host_string(); + let incoming = self.target.incoming_dir(); + let destination = format!("{host}:{incoming}/"); + let mut command = std::process::Command::new("scp"); + command + .arg("-p") + .args(self.target.hostkey_args()) + .args(self.target.port_args()); + for file in files { + command.arg(file); + } + command.arg(&destination); + let status = command + .status() + .with_context(|| format!("starting scp for {destination} failed"))?; + if !status.success() { + anyhow::bail!("scp failed (exit status: {status}) uploading to {destination}"); + } + Ok(()) + } +} diff --git a/packages/debmagic/src/upload/sftp.rs b/packages/debmagic/src/upload/sftp.rs new file mode 100644 index 00000000..0b74347a --- /dev/null +++ b/packages/debmagic/src/upload/sftp.rs @@ -0,0 +1,118 @@ +use std::io::{BufRead, BufReader}; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; + +use anyhow::Context; + +use super::{UploadTarget, Uploader}; +use crate::subprocess::{self, Capture}; + +pub struct SftpUploader { + pub target: UploadTarget, +} + +impl Uploader for SftpUploader { + fn upload_files(&mut self, files: &[PathBuf]) -> anyhow::Result<()> { + if files.is_empty() { + return Ok(()); + } + let host = self.target.host_string(); + // sftp has no `~` expansion; a path starting with `~user` is + // relative to that user's home on the server, which is exactly + // what the ppa builtin's `~{target}/ubuntu` relies on. + let incoming = self.target.incoming_dir(); + + // one batch of `put` commands uploads everything in a single + // sftp session instead of one connection per file + let mut batch = String::new(); + for file in files { + let name = file + .file_name() + .context("upload file has no file name")? + .to_string_lossy(); + batch.push_str(&format!( + "put \"{}\" \"{incoming}/{name}\"\n", + file.display() + )); + } + batch.push_str("pwd\n"); + + let mut cmd = Command::new("sftp"); + cmd.arg("-b") + .arg("-") + .args(self.target.hostkey_args()) + .args(self.target.port_args()) + .arg(&host) + // sftp forks ssh for the connection; a process group lets us + // tear down both instead of orphaning the ssh child + .process_group(0); + + let mut child = subprocess::command(cmd) + .input(batch.as_bytes()) + .capture(Capture::STDOUT) + .spawn() + .with_context(|| format!("starting sftp for {host}:{incoming} failed"))?; + + // stream the session output as it happens; each `sftp> put` echo + // marks a file being sent, the final `pwd` output means all puts + // succeeded and the server will never close the connection, so stop + // waiting and tear the session down + let mut stdout = BufReader::new(child.stdout.take().expect("stdout is piped")); + let mut done = false; + let mut line = String::new(); + loop { + line.clear(); + match stdout.read_line(&mut line) { + Ok(0) | Err(_) => break, + Ok(_) => { + if let Some(path) = line + .strip_prefix("sftp> put \"") + .and_then(|rest| rest.split('"').next()) + { + let index = files + .iter() + .position(|f| f.as_os_str() == path) + .map(|i| i + 1) + .unwrap_or_default(); + let name = Path::new(path) + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_else(|| path.to_string()); + println!("debmagic: uploading {name} ({index}/{})", files.len()); + } + if line.contains("Remote working directory:") { + done = true; + break; + } + } + } + } + if !done { + // the batch aborted before the sentinel: let sftp finish so + // the exit status carries the real error; stderr already + // reached the user's terminal + let status = child + .wait() + .with_context(|| format!("waiting for sftp to {host}:{incoming} failed"))?; + anyhow::bail!( + "sftp failed (exit status: {status}) uploading {} files to {host}:{incoming}", + files.len() + ); + } + terminate_session(&mut child, &host, &incoming)?; + Ok(()) + } +} + +/// Kill the sftp process group; the server will not close the connection +/// on its own, so this is the only way to end the session. +fn terminate_session(child: &mut Child, host: &str, incoming: &str) -> anyhow::Result<()> { + // SIGTERM to the group lets sftp and its ssh child close the channel + // cleanly; the id is ours because of `process_group(0)` + unsafe { libc::kill(-(child.id() as i32), libc::SIGTERM) }; + child + .wait() + .with_context(|| format!("terminating the sftp session for {host}:{incoming} failed"))?; + Ok(()) +} diff --git a/packages/debmagic/src/upload/ssh.rs b/packages/debmagic/src/upload/ssh.rs new file mode 100644 index 00000000..a6259fbc --- /dev/null +++ b/packages/debmagic/src/upload/ssh.rs @@ -0,0 +1,35 @@ +use super::UploadTarget; + +/// Shared ssh-based transport bits for the scp/sftp uploaders. +impl UploadTarget { + /// `login@server`, or just `server` when no login is configured + /// (the ssh config then decides the user). + pub fn host_string(&self) -> String { + match &self.config.login { + Some(login) => format!("{login}@{}", self.config.server), + None => self.config.server.clone(), + } + } + + /// `["-P", ""]` — scp and sftp both use `-P` for the port; + /// empty when unconfigured. + pub fn port_args(&self) -> Vec { + match self.config.port { + Some(port) => vec!["-P".to_string(), port.to_string()], + None => Vec::new(), + } + } + + /// `["-oStrictHostKeyChecking=accept-new"]` when TOFU host-key + /// trust is enabled for this target, else empty. sftp batch mode + /// cannot prompt for an unknown host key, so without this a first + /// upload to a new host fails with a bare "Host key verification + /// failed". + pub fn hostkey_args(&self) -> Vec { + if self.config.tofu_hostkey { + vec!["-oStrictHostKeyChecking=accept-new".to_string()] + } else { + Vec::new() + } + } +} diff --git a/packages/debmagic/src/upload/target.rs b/packages/debmagic/src/upload/target.rs new file mode 100644 index 00000000..5fbd1755 --- /dev/null +++ b/packages/debmagic/src/upload/target.rs @@ -0,0 +1,323 @@ +use std::collections::HashMap; + +use anyhow::bail; + +use super::UploadMethod; + +/// A named upload target definition, builtin or from the config. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct UploadTargetConfig { + /// Which upload method to use. `None` keeps a builtin's method + /// (or defaults to scp for config-only targets). + pub method: Option, + /// Host to upload to. + pub server: String, + /// Remote directory to upload into; supports `{target}`. + pub incoming: String, + /// Login on the remote host. `None` lets the ssh config decide, + /// then the local user — the Launchpad/Debian upload hosts expect + /// your own username with a registered SSH key. + pub login: Option, + /// Remote port. `None` lets the ssh config decide. + pub port: Option, + /// Trust the host key on first use (TOFU, ssh's `StrictHostKeyChecking=accept-new`). + /// The upload is signed via pgp for the real trust. + #[serde(default = "default_tofu_hostkey")] + pub tofu_hostkey: bool, + /// Commands to run before uploading, each via `sh -c`. + pub pre_upload_commands: Vec, +} + +fn default_tofu_hostkey() -> bool { + true +} + +impl UploadTargetConfig { + fn new( + method: UploadMethod, + server: &str, + incoming: &str, + login: Option<&str>, + port: Option, + ) -> Self { + Self { + method: Some(method), + server: server.to_string(), + incoming: incoming.to_string(), + login: login.map(str::to_string), + port, + tofu_hostkey: true, + pre_upload_commands: Vec::new(), + } + } +} + +/// One resolved upload target. +#[derive(Debug, Clone)] +pub struct UploadTarget { + pub name: String, + /// The parameter after `:` in `ppa:user/repo`-style invocations. + pub parameter: Option, + pub config: UploadTargetConfig, +} + +impl UploadTarget { + /// The remote directory to upload into, with `{target}` substituted. + pub fn incoming_dir(&self) -> String { + substitute_placeholders( + &self.config.incoming, + &[("target", self.parameter.as_deref().unwrap_or(""))], + ) + } +} + +/// Substitute `{name}` placeholders in `s`. +pub(crate) fn substitute_placeholders(s: &str, vars: &[(&str, &str)]) -> String { + let mut out = s.to_string(); + for (name, value) in vars { + let placeholder = format!("{{{name}}}"); + out = out.replace(&placeholder, value); + } + out +} + +/// Parse `name` or `name:parameter` (split on the first `:`). +pub fn parse_target_spec(spec: &str) -> (String, Option) { + match spec.split_once(':') { + Some((name, parameter)) => (name.to_string(), Some(parameter.to_string())), + None => (spec.to_string(), None), + } +} + +/// Built-in upload targets. Each can be overridden/extended +/// field-by-field via `[upload.targets.]` in the config file. +pub fn builtin_targets() -> Vec<(&'static str, UploadTargetConfig)> { + vec![ + ( + "ppa", + UploadTargetConfig::new( + UploadMethod::Sftp, + "ppa.launchpad.net", + "~{target}/ubuntu", + None, + None, + ), + ), + ( + "ubuntu", + UploadTargetConfig::new( + UploadMethod::Sftp, + "upload.ubuntu.com", + "ubuntu", + None, + None, + ), + ), + ( + "debian", + UploadTargetConfig::new( + UploadMethod::Sftp, + "ssh.upload.debian.org", + "/srv/upload.debian.org/UploadQueue", + None, + None, + ), + ), + ] +} + +/// Resolve an upload target spec against the config's `[upload.targets]` +/// table merged field-by-field over a same-named builtin. +pub fn resolve_target( + spec: &str, + config_targets: Option<&HashMap>, +) -> anyhow::Result { + let (name, parameter) = parse_target_spec(spec); + + let builtin = builtin_targets() + .into_iter() + .find(|(builtin_name, _)| *builtin_name == name) + .map(|(_, config)| config); + + let config = match (builtin, config_targets.and_then(|t| t.get(&name))) { + (Some(mut builtin), Some(overlay)) => { + merge_target_config(&mut builtin, overlay); + builtin + } + (Some(builtin), None) => builtin, + (None, Some(config)) => config.clone(), + (None, None) => bail!( + "unknown upload target '{name}'; configure [upload.targets.{name}] or use a builtin: ppa, ubuntu, debian" + ), + }; + + Ok(UploadTarget { + name, + parameter, + config, + }) +} + +/// Field-by-field merge of `overlay` into `base`; unset overlay fields +/// keep the builtin's values. +fn merge_target_config(base: &mut UploadTargetConfig, overlay: &UploadTargetConfig) { + if !overlay.server.is_empty() { + base.server = overlay.server.clone(); + } + if !overlay.incoming.is_empty() { + base.incoming = overlay.incoming.clone(); + } + if overlay.login.is_some() { + base.login = overlay.login.clone(); + } + if overlay.port.is_some() { + base.port = overlay.port; + } + if !overlay.pre_upload_commands.is_empty() { + base.pre_upload_commands = overlay.pre_upload_commands.clone(); + } + if overlay.method.is_some() { + base.method = overlay.method; + } + base.tofu_hostkey = overlay.tofu_hostkey; +} + +/// The `[upload]` config section. +#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize)] +#[serde(default)] +pub struct UploadConfig { + /// Named upload targets, merged over the builtins. + pub targets: HashMap, +} + +/// CLI overrides for one upload invocation, layered over the +/// resolved target (like `DriverOverrides` for builds). +#[derive(Debug, Default, Clone)] +pub struct UploadOverrides { + pub method: Option, + pub server: Option, + pub incoming: Option, + pub login: Option, + pub port: Option, +} + +impl UploadTarget { + pub fn apply_overrides(&mut self, overrides: &UploadOverrides) { + if let Some(method) = overrides.method { + self.config.method = Some(method); + } + if let Some(server) = &overrides.server { + self.config.server = server.clone(); + } + if let Some(incoming) = &overrides.incoming { + self.config.incoming = incoming.clone(); + } + if let Some(login) = &overrides.login { + self.config.login = Some(login.clone()); + } + if let Some(port) = overrides.port { + self.config.port = Some(port); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_target_spec() { + assert_eq!( + parse_target_spec("ppa:sfttech/debmagic"), + ("ppa".to_string(), Some("sfttech/debmagic".to_string())) + ); + assert_eq!(parse_target_spec("ubuntu"), ("ubuntu".to_string(), None)); + assert_eq!( + parse_target_spec("host:path:with:colons"), + ("host".to_string(), Some("path:with:colons".to_string())) + ); + } + + #[test] + fn test_substitute_placeholders() { + assert_eq!( + substitute_placeholders("~{target}/ubuntu", &[("target", "sfttech/debmagic")]), + "~sfttech/debmagic/ubuntu" + ); + assert_eq!( + substitute_placeholders("no placeholders", &[("target", "x")]), + "no placeholders" + ); + } + + #[test] + fn test_resolve_builtin() { + let target = resolve_target("debian", None).unwrap(); + assert_eq!(target.config.server, "ssh.upload.debian.org"); + assert_eq!(target.config.incoming, "/srv/upload.debian.org/UploadQueue"); + assert_eq!(target.config.method, Some(UploadMethod::Sftp)); + } + + #[test] + fn test_resolve_unknown() { + assert!(resolve_target("nonexistent", None).is_err()); + } + + #[test] + fn test_resolve_merge_over_builtin() { + let mut targets = HashMap::new(); + targets.insert( + "ubuntu".to_string(), + UploadTargetConfig::new(UploadMethod::Scp, "", "my-incoming", None, None), + ); + let target = resolve_target("ubuntu", Some(&targets)).unwrap(); + // server comes from the builtin, incoming/method from the config + assert_eq!(target.config.server, "upload.ubuntu.com"); + assert_eq!(target.config.incoming, "my-incoming"); + assert_eq!(target.config.method, Some(UploadMethod::Scp)); + } + + #[test] + fn test_resolve_unset_method_not_overriding() { + // an unset method field keeps the builtin's + let mut targets = HashMap::new(); + targets.insert( + "debian".to_string(), + UploadTargetConfig { + method: None, + server: "other.host".to_string(), + incoming: String::new(), + login: None, + port: None, + tofu_hostkey: true, + pre_upload_commands: Vec::new(), + }, + ); + let target = resolve_target("debian", Some(&targets)).unwrap(); + assert_eq!(target.config.method, Some(UploadMethod::Sftp)); + } + + #[test] + fn test_incoming_dir_parameter_substitution() { + let target = resolve_target("ppa:sfttech/debmagic", None).unwrap(); + assert_eq!(target.incoming_dir(), "~sfttech/debmagic/ubuntu"); + } + + #[test] + fn test_config_target_without_builtin() { + let mut targets = HashMap::new(); + targets.insert( + "myhost".to_string(), + UploadTargetConfig::new( + UploadMethod::Scp, + "example.com", + "/srv/incoming", + Some("sfttech"), + Some(2222), + ), + ); + let target = resolve_target("myhost", Some(&targets)).unwrap(); + assert_eq!(target.config.server, "example.com"); + assert_eq!(target.config.port, Some(2222)); + } +} diff --git a/packages/debmagic/src/upstream.rs b/packages/debmagic/src/upstream.rs new file mode 100644 index 00000000..aa47bcf4 --- /dev/null +++ b/packages/debmagic/src/upstream.rs @@ -0,0 +1,6 @@ +pub mod mangle; +pub mod orig; +pub mod query; +pub mod repack; +pub mod switch; +pub mod watch; diff --git a/packages/debmagic/src/upstream/mangle.rs b/packages/debmagic/src/upstream/mangle.rs new file mode 100644 index 00000000..ab90d4c6 --- /dev/null +++ b/packages/debmagic/src/upstream/mangle.rs @@ -0,0 +1,207 @@ +use anyhow::{Context, bail}; + +/// A `s/regex/replacement/flags` or `tr/src/dest/` (alias `y`) rule, +/// as used by the `*-Mangle` watch file options. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MangleRule { + Subst { + pattern: String, + replacement: String, + global: bool, + case_insensitive: bool, + }, + Transliterate { + from: String, + to: String, + }, +} + +/// Parse a mangle rule string: `;`-separated rules of +/// `s/pattern/replacement/[gi]` or `tr/from/to/` (`y/from/to/`), +/// with any non-alphanumeric delimiter. +pub fn parse_mangle_rules(rules: &str) -> anyhow::Result> { + let mut parsed = Vec::new(); + for rule in rules.split(';') { + let rule = rule.trim(); + if rule.is_empty() { + continue; + } + let mut chars = rule.chars(); + let op = chars.next().context("empty mangle rule")?; + match op { + 's' => { + let delimiter = chars.next().context("missing delimiter in s-rule")?; + if delimiter.is_alphanumeric() { + bail!("invalid delimiter {delimiter:?} in mangle rule: {rule}"); + } + let (pattern, rest) = split_on_delimiter(chars.as_str(), delimiter) + .with_context(|| format!("unterminated pattern in mangle rule: {rule}"))?; + let (replacement, flags) = split_on_delimiter(&rest, delimiter) + .with_context(|| format!("unterminated replacement in mangle rule: {rule}"))?; + let mut global = false; + let mut case_insensitive = false; + for flag in flags.trim().chars() { + match flag { + 'g' => global = true, + 'i' => case_insensitive = true, + _ => bail!("unsupported flag {flag:?} in mangle rule: {rule}"), + } + } + if pattern.is_empty() { + bail!("empty pattern in mangle rule: {rule}"); + } + parsed.push(MangleRule::Subst { + pattern, + replacement, + global, + case_insensitive, + }); + } + 't' => { + if chars.next() != Some('r') { + bail!("unknown mangle operation in rule: {rule}"); + } + parsed.push(parse_transliterate(chars.as_str(), rule)?); + } + 'y' => { + parsed.push(parse_transliterate(chars.as_str(), rule)?); + } + _ => bail!("unsupported mangle operation {op:?} in rule: {rule} (only s, tr, y)"), + } + } + if parsed.is_empty() { + bail!("no rules found in mangle string: {rules}"); + } + Ok(parsed) +} + +/// Split `s` at the next unescaped `delimiter`, returning (head, tail-after-delimiter). +/// Backslash-escaped delimiters are unescaped in the head. +fn split_on_delimiter(s: &str, delimiter: char) -> Option<(String, String)> { + let mut head = String::new(); + let mut chars = s.chars(); + loop { + let c = chars.next()?; + if c == '\\' { + let escaped = chars.next()?; + if escaped == delimiter { + head.push(delimiter); + } else { + head.push('\\'); + head.push(escaped); + } + } else if c == delimiter { + return Some((head, chars.as_str().to_string())); + } else { + head.push(c); + } + } +} + +fn parse_transliterate(s: &str, rule: &str) -> anyhow::Result { + let mut chars = s.chars(); + let delimiter = chars + .next() + .with_context(|| format!("missing delimiter in tr-rule: {rule}"))?; + if delimiter.is_alphanumeric() { + bail!("invalid delimiter {delimiter:?} in mangle rule: {rule}"); + } + let (from, rest) = split_on_delimiter(chars.as_str(), delimiter) + .with_context(|| format!("unterminated source in mangle rule: {rule}"))?; + let (to, _flags) = split_on_delimiter(&rest, delimiter) + .with_context(|| format!("unterminated destination in mangle rule: {rule}"))?; + Ok(MangleRule::Transliterate { from, to }) +} + +/// Apply parsed rules to `input` in order. +pub fn apply_mangle_rules(rules: &[MangleRule], input: &str) -> anyhow::Result { + let mut result = input.to_string(); + for rule in rules { + result = match rule { + MangleRule::Subst { + pattern, + replacement, + global, + case_insensitive, + } => { + let mut builder = regex::RegexBuilder::new(pattern); + builder.case_insensitive(*case_insensitive); + let regex = builder + .build() + .with_context(|| format!("invalid regex in mangle rule: {pattern}"))?; + if *global { + regex.replace_all(&result, replacement).into_owned() + } else { + regex.replace(&result, replacement).into_owned() + } + } + MangleRule::Transliterate { from, to } => { + let from_chars: Vec = from.chars().collect(); + let to_chars: Vec = to.chars().collect(); + if from_chars.len() < to_chars.len() { + bail!("tr source longer than destination: {from} -> {to}"); + } + // Perl tr semantics: a shorter destination repeats its last char + result + .chars() + .map(|c| match from_chars.iter().position(|&f| f == c) { + Some(i) => { + if i < to_chars.len() { + to_chars[i] + } else { + *to_chars.last().unwrap_or(&c) + } + } + None => c, + }) + .collect() + } + }; + } + Ok(result) +} + +/// Parse and apply in one step, the common case for watch file options. +pub fn mangle(rules: &str, input: &str) -> anyhow::Result { + let parsed = parse_mangle_rules(rules)?; + apply_mangle_rules(&parsed, input) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_case::test_case; + + #[test_case("s/foo/bar/", "foo foo", "bar foo"; "single substitution")] + #[test_case("s/foo/bar/g", "foo foo", "bar bar"; "global substitution")] + #[test_case("s/FOO/bar/i", "foo", "bar"; "case insensitive")] + #[test_case("s/a\\/b/X/", "a/b", "X"; "escaped delimiter")] + #[test_case("s/\\+dfsg\\d*$//", "2.03+dfsg", "2.03"; "dfsg strip")] + #[test_case( + "s/(\\d)[_\\.\\-\\+]?((?:RC|rc|pre|dev|beta|alpha)\\d*)$/$1~$2/", + "1.2.3rc1", + "1.2.3~rc1" + ; "uversion auto" + )] + #[test_case("s/a/X/;s/b/Y/", "ab", "XY"; "multiple rules")] + #[test_case("tr/abc/xyz/", "abc", "xyz"; "transliterate")] + #[test_case("y/abc/xyz/", "abc", "xyz"; "y alias")] + #[test_case("s%prdownload%download%", "prdownload", "download"; "percent delimiter")] + fn test_mangle(rules: &str, input: &str, expected: &str) { + assert_eq!(mangle(rules, input).unwrap(), expected); + } + + #[test_case("s/foo"; "unterminated")] + #[test_case("x/foo/bar/"; "unknown op")] + #[test_case("s//bar/"; "empty pattern")] + #[test_case("s/foo/bar/z"; "bad flag")] + fn test_mangle_rejects(rules: &str) { + assert!(mangle(rules, "foo").is_err()); + } + + #[test] + fn test_tr_last_char_repeated() { + // Perl tr semantics: a shorter destination repeats its last char + assert_eq!(mangle("tr/abc/x/", "cab").unwrap(), "xxx"); + } +} diff --git a/packages/debmagic/src/upstream/orig.rs b/packages/debmagic/src/upstream/orig.rs new file mode 100644 index 00000000..b7110e38 --- /dev/null +++ b/packages/debmagic/src/upstream/orig.rs @@ -0,0 +1,607 @@ +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use anyhow::{Context, bail}; + +use debmagic_common::debian::version::PackageVersion; +use debmagic_common::package::SourcePackage; + +/// How to fetch the `orig` tarball a `3.0 (quilt)` source build needs +/// when no local copy exists. +#[derive( + Debug, + Default, + Copy, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + clap::ValueEnum, +)] +#[serde(rename_all = "kebab-case")] +pub enum OrigTarballMethod { + /// The distro archives via Launchpad's API: Ubuntu first, then + /// Debian (which Launchpad mirrors). No configuration needed. + #[default] + Launchpad, + /// Debian's own archive pool (`deb.debian.org`), without Launchpad. + Debian, + /// Ubuntu's own archive pool (`archive.ubuntu.com`), without Launchpad. + Ubuntu, + /// A custom command from `orig_tarball.command`. + Custom, + /// Never fetch: build only with tarballs found locally. + Disabled, +} + +/// The `[orig_tarball]` config section. +#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize)] +#[serde(default)] +pub struct OrigTarballConfig { + /// Which method to use; unset means `launchpad`. + pub method: Option, + /// Command for `method = "custom"`, run via `sh -c` in the source + /// dir with placeholders substituted (including `{output_dir}`). + pub command: Option, + /// Mirror root for `method = "debian"`, like an apt sources entry + /// (`https://deb.debian.org/debian` by default); `/pool` is appended. + pub debian_mirror: Option, + /// Mirror root for `method = "ubuntu"` (`http://archive.ubuntu.com/ubuntu` + /// by default); `/pool` is appended. + pub ubuntu_mirror: Option, +} + +impl OrigTarballConfig { + /// The resolved method: the configured one, or the `launchpad` + /// default when unset. + fn method(&self) -> OrigTarballMethod { + self.method.unwrap_or_default() + } + + /// The pool root on a mirror: the configured mirror (or the distro + /// default), with `/pool` appended. + fn pool_root(configured: Option<&str>, default: &str) -> String { + format!( + "{}/pool", + configured.unwrap_or(default).trim_end_matches('/') + ) + } +} + +/// Substitute `{name}`, `{version}`, `{upstream_version}`, `{source_dir}` +/// and `{output_dir}` placeholders. +fn substitute_placeholders(s: &str, package: &SourcePackage, output_dir: &Path) -> String { + let version = package.version().to_string(); + let source_dir = package.source_dir().unwrap().to_string_lossy(); + let out = output_dir.to_string_lossy(); + crate::upload::target::substitute_placeholders( + s, + &[ + ("name", package.name()), + ("version", version.as_str()), + ("upstream_version", package.version().upstream_version()), + ("source_dir", &source_dir), + ("output_dir", &out), + ], + ) +} + +/// The download cache dir for one package version: +/// `/debmagic/orig///`. +fn cache_dir(name: &str, upstream_version: &str) -> Option { + dirs::cache_dir().map(|cache| { + cache + .join("debmagic") + .join("orig") + .join(name) + .join(upstream_version) + }) +} + +/// Locate the wanted orig tarball in `dir`: the main one, or a MUT +/// component's when `component` is set. +fn find_in_dir( + dir: &Path, + name: &str, + upstream_version: &str, + component: Option<&str>, +) -> Option { + match component { + Some(component) => { + debmagic_common::changes::find_component_in_dir(dir, name, upstream_version, component) + } + None => debmagic_common::changes::find_orig_in_dir(dir, name, upstream_version), + } +} + +/// Fetch the `orig` tarball for `package` into `output_dir`, unless a +/// suitable one is already there. Debian/Ubuntu developers conventionally +/// keep one next to the source tree (`../`), so an existing tarball there +/// is used as-is instead of fetching. +/// +/// Returns the path of the tarball to build with, or `None` for native +/// packages that need no `orig` tarball at all. +pub async fn fetch_orig_tarball( + config: &OrigTarballConfig, + package: &SourcePackage, + output_dir: &Path, +) -> anyhow::Result> { + fetch_orig_tarball_for_version( + config, + package, + output_dir, + package.version().upstream_version(), + None, + ) + .await +} + +/// [`fetch_orig_tarball`], for an explicit upstream version — used when +/// the version is not (yet) the changelog head, like `upstream switch` +/// operating on its candidate. `component` selects a MUT component's +/// tarball instead of the main one. +pub async fn fetch_orig_tarball_for_version( + config: &OrigTarballConfig, + package: &SourcePackage, + output_dir: &Path, + upstream_version: &str, + component: Option<&str>, +) -> anyhow::Result> { + if package.is_native() { + return Ok(None); + } + + let source_dir = package.source_dir()?; + let name = package.name(); + + if let Some(existing) = find_in_dir(output_dir, name, upstream_version, component) { + println!( + "debmagic: using existing orig tarball {}", + existing.display() + ); + return Ok(Some(existing)); + } + + // the conventional location developers keep tarballs in + if let Some(parent) = source_dir.parent() + && let Some(existing) = find_in_dir(parent, name, upstream_version, component) + { + println!( + "debmagic: using existing orig tarball {} from {}", + existing.display(), + parent.display() + ); + return Ok(Some(existing)); + } + + let method = config.method(); + if method == OrigTarballMethod::Disabled { + return Ok(None); + } + + std::fs::create_dir_all(output_dir) + .with_context(|| format!("failed to create {}", output_dir.display()))?; + + match method { + OrigTarballMethod::Launchpad => fetch_from_distro_archive( + package, + upstream_version, + component, + output_dir, + ) + .await + .with_context(|| { + format!( + "fetching {name} {upstream_version} from the distro archives via Launchpad failed" + ) + }), + OrigTarballMethod::Debian => fetch_from_pool( + &OrigTarballConfig::pool_root( + config.debian_mirror.as_deref(), + "https://deb.debian.org/debian", + ), + &["main", "contrib", "non-free"], + name, + upstream_version, + component, + output_dir, + ) + .await + .with_context(|| format!("fetching {name} {upstream_version} from the Debian pool failed")), + OrigTarballMethod::Ubuntu => fetch_from_pool( + &OrigTarballConfig::pool_root( + config.ubuntu_mirror.as_deref(), + "http://archive.ubuntu.com/ubuntu", + ), + &["main", "universe", "multiverse", "restricted"], + name, + upstream_version, + component, + output_dir, + ) + .await + .with_context(|| format!("fetching {name} {upstream_version} from the Ubuntu pool failed")), + OrigTarballMethod::Disabled => Ok(None), + OrigTarballMethod::Custom => { + let command = config.command.as_deref().context( + "orig_tarball.method = \"custom\" requires orig_tarball.command to be set", + )?; + let command = substitute_placeholders(command, package, output_dir); + println!("debmagic: running orig tarball command: {command}"); + let status = std::process::Command::new("sh") + .arg("-c") + .arg(&command) + .current_dir(source_dir) + .status() + .with_context(|| format!("failed to run orig tarball command: {command}"))?; + if !status.success() { + bail!("orig tarball command failed (exit status: {status}): {command}"); + } + find_in_dir(output_dir, name, upstream_version, component) + .with_context(|| { + format!( + "orig tarball method did not produce an expected tarball in {}", + output_dir.display() + ) + }) + .map(Some) + } + } +} + +/// The `launchpad` method: the download cache, then the distro +/// archives via Launchpad's `+files` download URLs, populating the +/// cache. +/// +/// The orig filename is deterministic — only the compression +/// extension varies — so the tarball URL is constructed directly and +/// probed, like the pool methods probe their listing. No API query is +/// needed, so an unpublished changelog head (the usual case: the +/// version being built is new) is not a problem: the orig tarball +/// only depends on the upstream part, and Launchpad serves it under +/// the upstream-only filename. +/// The Launchpad download URL of a source package's orig tarball in a +/// distro archive, trying the compression extensions dpkg accepts. +/// `None` when the archive does not serve it. +async fn launchpad_orig_url( + distro: &str, + name: &str, + upstream_version: &str, + component: Option<&str>, +) -> Option { + let base = format!("https://launchpad.net/{distro}/+archive/primary/+files/"); + let prefix = match component { + None => debmagic_common::changes::orig_prefix(name, upstream_version), + Some(component) => { + debmagic_common::changes::component_orig_prefix(name, upstream_version, component) + } + }; + // dpkg accepts exactly these compression extensions (its %COMP + // table), so nothing else can appear in the archive + for ext in ["gz", "xz", "bz2", "lzma"] { + let url = format!("{base}{prefix}{ext}"); + if crate::requests::http_exists(&url).await { + return Some(url); + } + } + None +} + +async fn fetch_from_distro_archive( + package: &SourcePackage, + upstream_version: &str, + component: Option<&str>, + output_dir: &Path, +) -> anyhow::Result> { + let name = package.name(); + let cache = cache_dir(name, upstream_version); + if let Some(cache_dir) = &cache + && let Some(cached) = find_in_dir(cache_dir, name, upstream_version, component) + { + println!("debmagic: using cached orig tarball {}", cached.display()); + return Ok(Some(stage_from_cache(&cached, output_dir)?)); + } + + // Ubuntu first, then Debian: Launchpad mirrors Debian, so a + // Debian-only version is found too. + let mut orig_url = None; + for distro in ["ubuntu", "debian"] { + if let Some(url) = launchpad_orig_url(distro, name, upstream_version, component).await { + orig_url = Some(url); + break; + } + } + let Some(orig_url) = orig_url else { + return Ok(None); + }; + let orig_name = orig_url.rsplit('/').next().unwrap_or_default(); + let orig_path = output_dir.join(orig_name); + println!("debmagic: downloading {orig_url}"); + crate::requests::http_download(&orig_url, &orig_path) + .await + .with_context(|| format!("downloading {orig_name} failed"))?; + + if let Some(cache_dir) = &cache { + let cached = cache_dir.join(orig_name); + if let Err(error) = + std::fs::create_dir_all(cache_dir).and_then(|()| std::fs::copy(&orig_path, &cached)) + { + // a failing cache must never fail the build + println!("debmagic: populating the orig cache failed: {error}"); + } + } + + Ok(Some(orig_path)) +} + +/// Make the cached tarball available in `output_dir` without copying +/// bytes when avoidable: a hardlink when both are on the same +/// filesystem, a copy otherwise. +fn stage_from_cache(cached: &Path, output_dir: &Path) -> anyhow::Result { + let destination = output_dir.join(cached.file_name().unwrap_or_default()); + if std::fs::hard_link(cached, &destination).is_ok() { + return Ok(destination); + } + std::fs::copy(cached, &destination).with_context(|| { + format!( + "failed to copy {} to {}", + cached.display(), + destination.display() + ) + })?; + Ok(destination) +} + +/// The `debian`/`ubuntu` methods: a distro archive pool directly, +/// without Launchpad. The pool directory is deterministic +/// (`/
///`); its listing reveals the orig +/// tarball and the `.dsc` for verification. Sections are tried in +/// order until one lists the package. +async fn fetch_from_pool( + pool_root: &str, + sections: &[&str], + name: &str, + upstream_version: &str, + component: Option<&str>, + output_dir: &Path, +) -> anyhow::Result> { + let cache = cache_dir(name, upstream_version); + if let Some(cache_dir) = &cache + && let Some(cached) = find_in_dir(cache_dir, name, upstream_version, component) + { + println!("debmagic: using cached orig tarball {}", cached.display()); + return Ok(Some(stage_from_cache(&cached, output_dir)?)); + } + + // the pool groups packages by source name prefix: `libfoo` under + // `libf`, everything else under its first letter + let prefix = if name.starts_with("lib") { + name.chars().take(4).collect::() + } else { + name.chars().take(1).collect::() + }; + + // the pool holds origs of many versions; the filename must carry + // the wanted one + let orig_prefix = match component { + None => debmagic_common::changes::orig_prefix(name, upstream_version), + Some(component) => { + debmagic_common::changes::component_orig_prefix(name, upstream_version, component) + } + }; + let wanted = |filename: &str| -> bool { + filename.starts_with(&orig_prefix) && debmagic_common::changes::is_orig_tarball(filename) + }; + + for section in sections { + let pool_url = format!("{pool_root}/{section}/{prefix}/{name}/"); + let Ok(listing) = crate::requests::http_get(&pool_url).await else { + continue; + }; + let hrefs = crate::upstream::query::extract_hrefs(&listing); + let Some(orig_name) = hrefs + .iter() + .map(String::as_str) + .find(|filename| wanted(filename)) + .map(str::to_string) + else { + continue; + }; + let orig_path = output_dir.join(&orig_name); + println!("debmagic: downloading {pool_url}{orig_name}"); + crate::requests::http_download(&format!("{pool_url}{orig_name}"), &orig_path) + .await + .with_context(|| format!("downloading {orig_name} from {pool_url} failed"))?; + + // verify against the .dsc of the newest matching publication + let dsc_name = hrefs + .iter() + .map(String::as_str) + .filter(|filename| filename.ends_with(".dsc")) + .filter(|filename| { + PackageVersion::from_str(filename.trim_end_matches(".dsc")) + .is_ok_and(|v| v.upstream_version() == upstream_version) + }) + .max_by(|a, b| { + let a = PackageVersion::from_str(a.trim_end_matches(".dsc")); + let b = PackageVersion::from_str(b.trim_end_matches(".dsc")); + match (a, b) { + (Ok(a), Ok(b)) => a.cmp(&b), + _ => std::cmp::Ordering::Equal, + } + }) + .map(str::to_string); + if let Some(dsc_name) = dsc_name { + let dsc_path = output_dir.join(format!("{name}.dsc")); + crate::requests::http_download(&format!("{pool_url}{dsc_name}"), &dsc_path) + .await + .with_context(|| format!("downloading {dsc_name} from {pool_url} failed"))?; + let dsc = crate::control::read_control(&dsc_path)?; + let digests = crate::control::digest_file(&orig_path)?; + debmagic_common::debian::control::verify_checksums(&dsc, &orig_name, &digests) + .with_context(|| format!("the downloaded {orig_name} does not match its .dsc"))?; + std::fs::remove_file(&dsc_path).ok(); + } + + if let Some(cache_dir) = &cache { + let cached = cache_dir.join(&orig_name); + if let Err(error) = + std::fs::create_dir_all(cache_dir).and_then(|()| std::fs::copy(&orig_path, &cached)) + { + // a failing cache must never fail the build + println!("debmagic: populating the orig cache failed: {error}"); + } + } + + return Ok(Some(orig_path)); + } + Ok(None) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A package over a real temp dir, so the reader exercises the + /// on-disk path the production code uses. + fn package(dir: &Path, format: Option<&str>) -> SourcePackage { + std::fs::create_dir_all(dir.join("debian").join("source")).unwrap(); + std::fs::write( + dir.join("debian").join("changelog"), + "postfix (3.11.7-1ubuntu2) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n", + ) + .unwrap(); + if let Some(format) = format { + std::fs::write(dir.join("debian").join("source").join("format"), format).unwrap(); + } + crate::package::load_package(dir).unwrap() + } + + #[tokio::test] + async fn test_native_package_needs_no_orig() { + let base = std::env::temp_dir().join(format!("debmagic-native-{}", std::process::id())); + let source = base.join("pkg"); + let pkg = package(&source, Some("3.0 (native)\n")); + // a stale orig tarball next to the source must not be picked up + std::fs::write(base.join("pkg_1.0.orig.tar.gz"), "stale").unwrap(); + + let config = OrigTarballConfig::default(); + assert_eq!( + fetch_orig_tarball(&config, &pkg, &base).await.unwrap(), + None + ); + std::fs::remove_dir_all(&base).unwrap(); + } + + #[tokio::test] + async fn test_fetch_prefers_output_dir() { + let base = std::env::temp_dir().join(format!("debmagic-orig-{}", std::process::id())); + let source = base.join("postfix"); + let out = base.join("out"); + std::fs::create_dir_all(&out).unwrap(); + let pkg = package(&source, None); + // a tarball in the output dir wins over the parent dir one + std::fs::write(base.join("postfix_3.11.7.orig.tar.xz"), "parent").unwrap(); + std::fs::write(out.join("postfix_3.11.7.orig.tar.gz"), "output").unwrap(); + + let config = OrigTarballConfig::default(); + let found = fetch_orig_tarball(&config, &pkg, &out).await.unwrap(); + assert_eq!(found, Some(out.join("postfix_3.11.7.orig.tar.gz"))); + std::fs::remove_dir_all(&base).unwrap(); + } + + #[tokio::test] + async fn test_fetch_uses_existing_in_parent() { + let base = + std::env::temp_dir().join(format!("debmagic-orig-parent-{}", std::process::id())); + let source = base.join("postfix"); + let out = base.join("out"); + std::fs::create_dir_all(&out).unwrap(); + let pkg = package(&source, None); + // tarball next to the source tree, like developers keep it + std::fs::write(base.join("postfix_3.11.7.orig.tar.xz"), "x").unwrap(); + + let config = OrigTarballConfig::default(); + let found = fetch_orig_tarball(&config, &pkg, &out).await.unwrap(); + assert_eq!(found, Some(base.join("postfix_3.11.7.orig.tar.xz"))); + // nothing was copied into the output dir + assert!(out.read_dir().unwrap().next().is_none()); + std::fs::remove_dir_all(&base).unwrap(); + } + + #[tokio::test] + async fn test_fetch_no_method_no_tarball() { + let base = std::env::temp_dir().join(format!("debmagic-orig-none-{}", std::process::id())); + let source = base.join("postfix"); + let pkg = package(&source, None); + // fetching disabled explicitly: no method runs, no tarball appears + let config = OrigTarballConfig { + method: Some(OrigTarballMethod::Disabled), + ..Default::default() + }; + assert_eq!( + fetch_orig_tarball(&config, &pkg, &base).await.unwrap(), + None + ); + std::fs::remove_dir_all(&base).unwrap(); + } + + #[tokio::test] + async fn test_fetch_component_uses_component_tarball() { + let base = std::env::temp_dir().join(format!("debmagic-orig-comp-{}", std::process::id())); + let source = base.join("postfix"); + let out = base.join("out"); + std::fs::create_dir_all(&out).unwrap(); + let pkg = package(&source, None); + // only a component orig exists; the main lookup must not find it + std::fs::write(out.join("postfix_3.11.7.orig-bar.tar.xz"), "x").unwrap(); + + let config = OrigTarballConfig::default(); + let found = fetch_orig_tarball_for_version(&config, &pkg, &out, "3.11.7", Some("bar")) + .await + .unwrap(); + assert_eq!(found, Some(out.join("postfix_3.11.7.orig-bar.tar.xz"))); + std::fs::remove_dir_all(&base).unwrap(); + } + + #[test] + fn test_stage_from_cache_hardlinks() { + let base = std::env::temp_dir().join(format!("debmagic-orig-link-{}", std::process::id())); + let cache = base.join("cache"); + let out = base.join("out"); + std::fs::create_dir_all(&cache).unwrap(); + std::fs::create_dir_all(&out).unwrap(); + let cached = cache.join("postfix_3.11.7.orig.tar.xz"); + std::fs::write(&cached, "x").unwrap(); + + let staged = stage_from_cache(&cached, &out).unwrap(); + use std::os::unix::fs::MetadataExt; + assert_eq!( + std::fs::metadata(&cached).unwrap().ino(), + std::fs::metadata(&staged).unwrap().ino() + ); + std::fs::remove_dir_all(&base).unwrap(); + } + + #[test] + fn test_substitute_placeholders() { + let base = std::env::temp_dir().join(format!("debmagic-orig-subst-{}", std::process::id())); + let source = base.join("postfix"); + let pkg = package(&source, None); + let out = base.join("out"); + assert_eq!( + substitute_placeholders( + "fetch {name} {version} {upstream_version} from {source_dir} to {output_dir}", + &pkg, + &out + ), + format!( + "fetch postfix 3.11.7-1ubuntu2 3.11.7 from {} to {}", + source.display(), + out.display() + ) + ); + std::fs::remove_dir_all(&base).unwrap(); + } +} diff --git a/packages/debmagic/src/upstream/query.rs b/packages/debmagic/src/upstream/query.rs new file mode 100644 index 00000000..616f2711 --- /dev/null +++ b/packages/debmagic/src/upstream/query.rs @@ -0,0 +1,650 @@ +use std::path::Path; +use std::str::FromStr; + +use anyhow::{Context, bail}; +use debmagic_common::debian::version::PackageVersion; +use regex::Regex; + +use crate::upstream::mangle::mangle; +use crate::upstream::watch::{SearchMode, WatchSource}; + +fn substitute(pattern: &str, package: &str) -> String { + pattern + .replace("@PACKAGE@", ®ex::escape(package)) + .replace( + "@ANY_VERSION@", + r"[-_]?[Vv]?(\d[\-+\.:\~\da-zA-Z]*)", + ) + .replace( + "@SEMANTIC_VERSION@", + r"[-_]?[Vv]?((?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?:[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)", + ) + .replace("@STABLE_VERSION@", r"[-_]?[Vv]?((?:[1-9]\d*)(?:\.\d+){2})") + .replace( + "@ARCHIVE_EXT@", + r"(?i)(?:\.(?:tar\.xz|tar\.bz2|tar\.gz|tar\.zstd?|zip|tgz|tbz|txz))", + ) + .replace( + "@SIGNATURE_EXT@", + r"(?i)(?:\.(?:tar\.xz|tar\.bz2|tar\.gz|tar\.zstd?|zip|tgz|tbz|txz))'(?:\.(?:asc|pgp|gpg|sig|sign))'", + ) + .replace("@DEB_EXT@", r"[\+~](?:debian|dfsg|ds|deb)(?:\.)?(?:\d+)?$") +} + +/// Extract the href values from an HTML listing. +pub fn extract_hrefs(html: &str) -> Vec { + let mut hrefs = Vec::new(); + let mut rest = html; + while let Some(start) = rest.find("href") { + rest = &rest[start..]; + let Some(open) = rest.find(['"', '\'']) else { + break; + }; + let quote = rest.as_bytes()[open] as char; + let after_open = &rest[open + 1..]; + let Some(close) = after_open.find(quote) else { + break; + }; + hrefs.push(after_open[..close].to_string()); + rest = &after_open[close + 1..]; + } + hrefs +} + +/// A candidate upstream release found by a query. +#[derive(Debug, Clone)] +pub struct Candidate { + pub version: String, + pub href: String, +} + +/// The archive extensions `@ARCHIVE_EXT@` stands for, in probe order. +const ARCHIVE_EXTS: &[&str] = &[ + "tar.gz", "tar.xz", "tar.bz2", "tar.zst", "zip", "tgz", "tbz", "txz", +]; + +/// The `[-_]?[Vv]?` prefixes the version macros may carry in filenames. +const VERSION_PREFIXES: &[&str] = &["", "v", "-v", "_v", "V", "-V", "_V", "-", "_"]; + +/// Turn regex literal escapes (`\.` → `.`) into plain text for template +/// construction. +fn unescape_regex_literals(s: &str) -> String { + let mut out = String::new(); + let mut chars = s.chars(); + while let Some(c) = chars.next() { + if c == '\\' + && let Some(next) = chars.next() + && !next.is_alphanumeric() + { + out.push(next); + } else { + out.push(c); + } + } + out +} + +/// Whether the pattern, outside its macros, is plain literal text — +/// character classes, quantifiers or alternations make the template +/// ambiguous and are not invertible. +fn is_literal_template(template: &str) -> bool { + !template.contains(['[', ']', '{', '}', '(', ')', '|', '*', '+', '?']) +} + +/// Construct the hrefs a concrete `version` would have, directly from +/// the watch pattern — the pattern is a URL template. Returns `None` +/// when the pattern is not invertible: a `Uversion-Mangle` (not +/// invertible), no version macro, or more than one capture group +/// (uscan joins groups with `.`; only one group is unambiguous). +/// Each constructed href must self-check: match the anchored pattern +/// and capture exactly `version`. `preferred_ext` (the extension of +/// the project's existing orig tarball, if any) is ordered first, so +/// the common case probes once. +fn construct_hrefs( + source: &WatchSource, + package: &str, + version: &str, + preferred_ext: Option<&str>, +) -> anyhow::Result>> { + let Some(hrefs) = construct_hrefs_inner(source, package, version, preferred_ext)? else { + return Ok(None); + }; + Ok(Some(hrefs)) +} + +/// [`construct_hrefs`] with a learned example: `learned` is a concrete +/// href from the listing (any version). Its prefix/extension shape +/// is extracted by matching it against the pattern and replacing the +/// captured version back with the requested one — so even patterns +/// pure inversion rejects (multi-group, non-literal) construct once +/// one listed sibling matches. +fn construct_hrefs_from_example( + source: &WatchSource, + package: &str, + version: &str, + learned: &str, +) -> anyhow::Result> { + let pattern = substitute(&source.matching_pattern, package); + let regex = Regex::new(&format!("^{pattern}$")) + .with_context(|| format!("invalid matching pattern: {pattern}"))?; + let Some(captures) = regex.captures(learned) else { + return Ok(None); + }; + // the learned href's version spans all capture groups (uscan joins + // them with `.`); replace the whole span with the requested version + let Some(first) = captures.get(1) else { + return Ok(None); + }; + let last = captures + .get(captures.len() - 1) + .expect("capture group 1 exists, so does the last"); + let href = format!( + "{}{version}{}", + &learned[..first.start()], + &learned[last.end()..] + ); + // self-check: the constructed href must match and capture the + // requested version + let Some(check) = regex.captures(&href) else { + return Ok(None); + }; + let mut joined = String::new(); + for group in 1..check.len() { + if let Some(part) = check.get(group) { + if !joined.is_empty() { + joined.push('.'); + } + joined.push_str(part.as_str()); + } + } + if joined != version { + return Ok(None); + } + Ok(Some(href)) +} + +fn construct_hrefs_inner( + source: &WatchSource, + package: &str, + version: &str, + preferred_ext: Option<&str>, +) -> anyhow::Result>> { + if source.uversion_mangle.is_some() { + return Ok(None); + } + let raw = &source.matching_pattern; + if !raw.contains("@ANY_VERSION@") + && !raw.contains("@SEMANTIC_VERSION@") + && !raw.contains("@STABLE_VERSION@") + { + return Ok(None); + } + let pattern = substitute(raw, package); + let regex = Regex::new(&format!("^{pattern}$")) + .with_context(|| format!("invalid matching pattern: {pattern}"))?; + // captures_len() counts the implicit group 0: exactly one capture + // group means the whole version sits in one place + if regex.captures_len() != 2 { + return Ok(None); + } + + let template = unescape_regex_literals(raw); + if !is_literal_template(&template) { + return Ok(None); + } + let mut exts: Vec = if raw.contains("@ARCHIVE_EXT@") { + ARCHIVE_EXTS.iter().map(|e| format!(".{e}")).collect() + } else { + vec![String::new()] + }; + // the project's own orig tarball extension is the strongest hint + // for what upstream uses; probe it first + if let Some(preferred) = preferred_ext + && let Some(index) = exts.iter().position(|e| e == preferred) + { + let hint = exts.remove(index); + exts.insert(0, hint); + } + + let mut hrefs = Vec::new(); + for ext in &exts { + for prefix in VERSION_PREFIXES { + let href = template + .replace("@PACKAGE@", package) + .replace("@ANY_VERSION@", &format!("{prefix}{version}")) + .replace("@SEMANTIC_VERSION@", &format!("{prefix}{version}")) + .replace("@STABLE_VERSION@", &format!("{prefix}{version}")) + .replace("@ARCHIVE_EXT@", ext); + let Some(captures) = regex.captures(&href) else { + continue; + }; + if captures.get(1).map(|c| c.as_str()) != Some(version) { + continue; + } + if !hrefs.contains(&href) { + hrefs.push(href); + } + } + } + Ok(Some(hrefs)) +} + +/// Resolve the candidate for a concrete version without scraping the +/// listing: construct the possible hrefs from the watch pattern and +/// probe them. `requested` may be a full debian version — only its +/// upstream part (with `Dversion-Mangle` applied) is used. The +/// extension of an existing orig tarball (output dir, next to the +/// tree, or the cache) is probed first. When no constructed URL +/// exists, the listing is fetched as a *structure teacher*: any +/// listed sibling reveals the prefix/extension shape, and the +/// requested version's URL is constructed from it — which also finds +/// versions that have already fallen out of the listing. Returns +/// `None` when neither works; callers fall back to filtering the +/// listing then. +pub async fn resolve_concrete( + source: &WatchSource, + package: &str, + requested: &str, + existing_orig: Option<&Path>, +) -> anyhow::Result> { + // probing goes over http; ftp listings have no HEAD equivalent + if source.source.starts_with("ftp://") || source.source.starts_with("ftps://") { + return Ok(None); + } + let version = current_upstream_version(source, requested)?; + let preferred_ext = existing_orig.and_then(|orig| { + orig.file_name() + .map(|n| n.to_string_lossy().into_owned()) + .and_then(|name| { + name.split_once(".tar.") + .map(|(_, ext)| format!(".tar.{ext}")) + }) + }); + let hrefs = + construct_hrefs(source, package, &version, preferred_ext.as_deref())?.unwrap_or_default(); + + // probe the constructed hrefs + for href in &hrefs { + let candidate = Candidate { + version: version.clone(), + href: href.clone(), + }; + let url = crate::upstream::switch::download_url(source, &candidate)?; + if crate::requests::http_exists(&url).await { + return Ok(Some(candidate)); + } + } + + // nothing constructed exists: let the listing teach the file + // structure — any listed sibling reveals prefix/extension, and + // the requested version's URL is constructed from that shape + let candidates = query_source(source, package).await?; + if let Some(example) = candidates.first() + && let Some(href) = construct_hrefs_from_example(source, package, &version, &example.href)? + { + let candidate = Candidate { + version: version.clone(), + href, + }; + let url = crate::upstream::switch::download_url(source, &candidate)?; + if crate::requests::http_exists(&url).await { + return Ok(Some(candidate)); + } + } + Ok(None) +} + +/// Query a watch source: fetch its page and return candidate +/// versions, sorted newest first. +pub async fn query_source(source: &WatchSource, package: &str) -> anyhow::Result> { + let pattern = substitute(&source.matching_pattern, package); + // uscan anchors the pattern (^...$): a substring match would also + // pick up sibling files like foo-1.0.tar.gz.sha256 + let anchored = format!("^{pattern}$"); + let regex = + Regex::new(&anchored).with_context(|| format!("invalid matching pattern: {pattern}"))?; + + let page = fetch_page(&source.source).await?; + let is_ftp = source.source.starts_with("ftp://") || source.source.starts_with("ftps://"); + let subjects: Vec = match source.search_mode { + SearchMode::Html if is_ftp => { + // ftp listings are `ls -l` style; the filename is the last token + page.lines() + .filter_map(|line| line.split_whitespace().next_back()) + .map(str::to_string) + .collect() + } + SearchMode::Html => extract_hrefs(&page), + SearchMode::Plain => vec![page], + }; + + let mut candidates: Vec = Vec::new(); + for subject in subjects { + let Some(captures) = regex.captures(&subject) else { + continue; + }; + // capture groups join with `.` + let mut version = String::new(); + for group in 1..captures.len() { + if let Some(part) = captures.get(group) { + if !version.is_empty() { + version.push('.'); + } + version.push_str(part.as_str()); + } + } + if version.is_empty() { + continue; + } + if let Some(rules) = &source.uversion_mangle { + version = mangle(rules, &version)?; + } + candidates.push(Candidate { + version, + href: subject, + }); + } + + candidates.dedup_by(|a, b| a.version == b.version); + + candidates.sort_by(|a, b| { + let a = PackageVersion::from_str(&a.version); + let b = PackageVersion::from_str(&b.version); + match (a, b) { + (Ok(a), Ok(b)) => b.cmp(&a), + // unparseable versions sink to the bottom + (Ok(_), Err(_)) => std::cmp::Ordering::Less, + (Err(_), Ok(_)) => std::cmp::Ordering::Greater, + (Err(_), Err(_)) => std::cmp::Ordering::Equal, + } + }); + Ok(candidates) +} + +/// Fetch a watch source's page over http(s) or ftp. +async fn fetch_page(url: &str) -> anyhow::Result { + if url.starts_with("ftp://") || url.starts_with("ftps://") { + crate::requests::ftp_get(url).await + } else { + crate::requests::http_get(url).await + } +} + +/// The changelog's current upstream version, with the source's +/// `dversion_mangle` rules applied for comparison against candidates. +pub fn current_upstream_version( + source: &WatchSource, + changelog_version: &str, +) -> anyhow::Result { + let version = PackageVersion::from_str(changelog_version) + .map_err(|_| anyhow::anyhow!("invalid changelog version: {changelog_version}"))?; + let upstream = version.upstream_version().to_string(); + match &source.dversion_mangle { + Some(rules) => mangle(rules, &upstream), + None => Ok(upstream), + } +} + +/// Find the candidate for a requested version. The request may be a +/// full debian version like `3.2.23-1` — only its upstream part is +/// used, since candidates carry pure upstream versions. +pub fn find_candidate<'a>(candidates: &'a [Candidate], requested: &str) -> Option<&'a Candidate> { + let upstream = PackageVersion::from_str(requested) + .ok() + .map(|v| v.upstream_version().to_string()) + .unwrap_or_else(|| requested.to_string()); + candidates.iter().find(|c| c.version == upstream) +} + +pub fn load_watch(source_dir: &Path) -> anyhow::Result> { + let watch_path = source_dir.join("debian").join("watch"); + if !watch_path.is_file() { + bail!( + "{} has no debian/watch file; declare the upstream source in debmagic.toml instead", + source_dir.display() + ); + } + crate::upstream::watch::parse_watch_file(&watch_path) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_substitute_any_version() { + let pattern = substitute("foo-@ANY_VERSION@@ARCHIVE_EXT@", "foo"); + let regex = Regex::new(&pattern).unwrap(); + let captures = regex.captures("foo-1.2.3.tar.gz").unwrap(); + assert_eq!(captures.get(1).unwrap().as_str(), "1.2.3"); + } + + #[test] + fn test_construct_hrefs_simple() { + let source = WatchSource { + matching_pattern: "haproxy-@ANY_VERSION@@ARCHIVE_EXT@".to_string(), + ..Default::default() + }; + let hrefs = construct_hrefs(&source, "haproxy", "3.2.23", None) + .unwrap() + .unwrap(); + // the plain form probes first, prefix variants after it + assert_eq!(hrefs.first().unwrap(), "haproxy-3.2.23.tar.gz"); + assert!(hrefs.contains(&"haproxy-3.2.23.tar.xz".to_string())); + // the short forms (tgz/tbz/txz) are alternates of the long ones + // and probe too — the first existing URL wins + assert!(hrefs.contains(&"haproxy-3.2.23.tgz".to_string())); + } + + #[test] + fn test_construct_hrefs_prefers_existing_ext() { + let source = WatchSource { + matching_pattern: "haproxy-@ANY_VERSION@@ARCHIVE_EXT@".to_string(), + ..Default::default() + }; + // the project's own orig is xz: that extension probes first + let hrefs = construct_hrefs(&source, "haproxy", "3.2.23", Some(".tar.xz")) + .unwrap() + .unwrap(); + assert_eq!(hrefs.first().unwrap(), "haproxy-3.2.23.tar.xz"); + assert_eq!(hrefs.get(1).unwrap(), "haproxy-v3.2.23.tar.xz"); + } + + #[test] + fn test_construct_from_example() { + // a listed sibling teaches the shape: prefix and extension + let source = WatchSource { + matching_pattern: "haproxy-@ANY_VERSION@@ARCHIVE_EXT@".to_string(), + ..Default::default() + }; + let href = + construct_hrefs_from_example(&source, "haproxy", "3.2.23", "haproxy-v3.2.24.tar.xz") + .unwrap() + .unwrap(); + assert_eq!(href, "haproxy-v3.2.23.tar.xz"); + + // a pattern pure inversion rejects (two capture groups) still + // constructs from an example + let source = WatchSource { + matching_pattern: r"foo-v(\d+)\.(\d+)@ARCHIVE_EXT@".to_string(), + ..Default::default() + }; + assert!( + construct_hrefs(&source, "foo", "1.0", None) + .unwrap() + .is_none() + ); + let href = construct_hrefs_from_example(&source, "foo", "2.0", "foo-v1.0.tar.gz") + .unwrap() + .unwrap(); + assert_eq!(href, "foo-v2.0.tar.gz"); + + // an example that does not match the pattern teaches nothing + let source = WatchSource { + matching_pattern: "haproxy-@ANY_VERSION@@ARCHIVE_EXT@".to_string(), + ..Default::default() + }; + assert!( + construct_hrefs_from_example(&source, "haproxy", "3.2.23", "other-1.0.tar.gz") + .unwrap() + .is_none() + ); + } + + #[test] + fn test_construct_hrefs_version_prefix() { + // the [-_]?[Vv]? prefix: v-prefixed releases construct too + let source = WatchSource { + matching_pattern: "foo-@ANY_VERSION@@ARCHIVE_EXT@".to_string(), + ..Default::default() + }; + let hrefs = construct_hrefs(&source, "foo", "1.0", None) + .unwrap() + .unwrap(); + assert!(hrefs.contains(&"foo-v1.0.tar.gz".to_string())); + assert!(hrefs.contains(&"foo-1.0.tar.gz".to_string())); + } + + #[test] + fn test_construct_hrefs_dotted_pattern() { + // regex literal escapes are unescaped for the template + let source = WatchSource { + matching_pattern: r"foo_@ANY_VERSION@\.tar\.gz".to_string(), + ..Default::default() + }; + let hrefs = construct_hrefs(&source, "foo", "1.2.3", None) + .unwrap() + .unwrap(); + assert!(hrefs.contains(&"foo_1.2.3.tar.gz".to_string())); + + // a character class outside the macros: not invertible + let source = WatchSource { + matching_pattern: r"foo[0-9.]+_@ANY_VERSION@\.tar\.gz".to_string(), + ..Default::default() + }; + assert!( + construct_hrefs(&source, "foo", "1.2.3", None) + .unwrap() + .is_none() + ); + } + + #[test] + fn test_construct_hrefs_not_invertible() { + // no version macro + let source = WatchSource { + matching_pattern: "foo-latest.tar.gz".to_string(), + ..Default::default() + }; + assert!( + construct_hrefs(&source, "foo", "1.0", None) + .unwrap() + .is_none() + ); + + // uversion_mangle: not invertible + let source = WatchSource { + matching_pattern: "foo-@ANY_VERSION@@ARCHIVE_EXT@".to_string(), + uversion_mangle: Some("s/$/+dfsg/".to_string()), + ..Default::default() + }; + assert!( + construct_hrefs(&source, "foo", "1.0", None) + .unwrap() + .is_none() + ); + + // more than one capture group: ambiguous how the version splits + let source = WatchSource { + matching_pattern: "foo-(\\d+)\\.(\\d+)@ARCHIVE_EXT@".to_string(), + ..Default::default() + }; + assert!( + construct_hrefs(&source, "foo", "1.0", None) + .unwrap() + .is_none() + ); + } + + #[test] + fn test_extract_hrefs() { + let html = r#"x y"#; + assert_eq!( + extract_hrefs(html), + vec!["a-1.0.tar.gz".to_string(), "b-2.0.tar.gz".to_string()] + ); + } + + #[test] + fn test_version_concatenation() { + // multiple capture groups join with `.` + let pattern = substitute("foo_v(\\d+)_(\\d+)@ARCHIVE_EXT@", "foo"); + let regex = Regex::new(&pattern).unwrap(); + let captures = regex.captures("foo_v1_2.tar.gz").unwrap(); + let mut version = String::new(); + for group in 1..captures.len() { + if let Some(part) = captures.get(group) { + if !version.is_empty() { + version.push('.'); + } + version.push_str(part.as_str()); + } + } + assert_eq!(version, "1.2"); + } + + #[test] + fn test_current_upstream_version_dfsg() { + let source = WatchSource { + dversion_mangle: Some("s/\\+dfsg\\d*$//".to_string()), + ..Default::default() + }; + assert_eq!( + current_upstream_version(&source, "2.03+dfsg-4").unwrap(), + "2.03" + ); + } + #[test] + fn test_pattern_anchored_no_sibling_files() { + // a .sha256 sibling of the tarball must not match the pattern + let source = WatchSource { + source: "https://example.com/".to_string(), + matching_pattern: "foo-@ANY_VERSION@@ARCHIVE_EXT@".to_string(), + ..Default::default() + }; + let pattern = substitute(&source.matching_pattern, "foo"); + let regex = Regex::new(&format!("^{pattern}$")).unwrap(); + assert!(regex.captures("foo-1.0.tar.gz").is_some()); + assert!(regex.captures("foo-1.0.tar.gz.sha256").is_none()); + assert!(regex.captures("foo-1.0.tar.gz.asc").is_none()); + } + + #[test] + fn test_find_candidate_accepts_debian_version() { + let candidates = vec![ + Candidate { + version: "3.2.22".to_string(), + href: "old".to_string(), + }, + Candidate { + version: "3.2.23".to_string(), + href: "new".to_string(), + }, + ]; + // a full debian version: only the upstream part is used + let found = find_candidate(&candidates, "3.2.23-1").unwrap(); + assert_eq!(found.version, "3.2.23"); + // a plain upstream version still works + assert_eq!( + find_candidate(&candidates, "3.2.22").unwrap().version, + "3.2.22" + ); + // an epoch is stripped too + assert_eq!( + find_candidate(&candidates, "1:3.2.23-1").unwrap().version, + "3.2.23" + ); + // unknown versions find nothing + assert!(find_candidate(&candidates, "9.9.9").is_none()); + } +} diff --git a/packages/debmagic/src/upstream/repack.rs b/packages/debmagic/src/upstream/repack.rs new file mode 100644 index 00000000..4089f444 --- /dev/null +++ b/packages/debmagic/src/upstream/repack.rs @@ -0,0 +1,181 @@ +use std::path::Path; + +use anyhow::Context; + +/// The repack configuration for one source: which files to strip from +/// the upstream tree and which version suffix the repack warrants. +#[derive(Debug, Default, Clone)] +pub struct RepackConfig { + /// Glob patterns to exclude, from `Files-Excluded` in + /// `debian/copyright` and the watch file's `Repack`/`Repacksuffix`. + pub excludes: Vec, + /// Suffix appended to the upstream version, e.g. `+dfsg1`. + pub suffix: Option, +} + +/// Load the repack configuration from the package's `debian/copyright` +/// (`Files-Excluded`, and `Files-Excluded-` for MUT +/// packages) merged with the watch source's repack options. +pub fn load_repack_config( + package: &debmagic_common::package::SourcePackage, + watch: &crate::upstream::watch::WatchSource, +) -> anyhow::Result { + let excluded = package.copyright_excludes()?; + let mut excludes = excluded.main; + match &watch.component { + // a MUT source only excludes its own component's patterns + Some(component) => excludes.extend( + excluded + .components + .get(component) + .cloned() + .unwrap_or_default(), + ), + // a non-component source applies every field, as uscan does + None => excludes.extend(excluded.components.into_values().flatten()), + } + + if watch.repack && excludes.is_empty() { + anyhow::bail!( + "the watch file requests a repack but debian/copyright has no Files-Excluded" + ); + } + Ok(RepackConfig { + excludes, + suffix: watch.repack_suffix.clone(), + }) +} + +/// Whether any exclude pattern matches `path` (relative, forward slashes). +/// A `dir/*` pattern excludes everything under `dir`, at any depth. +fn is_excluded(path: &str, excludes: &[String]) -> bool { + excludes.iter().any(|pattern| { + let Ok(glob) = glob::Pattern::new(pattern) else { + return false; + }; + if glob.matches(path) { + return true; + } + // the pattern may match an ancestor: `win32/*` covers win32/nested/b.c + let mut ancestor = String::new(); + path.split('/').any(|part| { + ancestor.push_str(part); + ancestor.push('/'); + glob.matches(&ancestor) + }) + }) +} + +/// Remove all files matching the exclude patterns from `tree_dir`. +/// Returns the number of removed files. +pub fn apply_excludes(tree_dir: &Path, excludes: &[String]) -> anyhow::Result { + let mut removed = 0; + let mut stack = vec![tree_dir.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in + std::fs::read_dir(&dir).with_context(|| format!("failed to read {}", dir.display()))? + { + let entry = entry?; + let path = entry.path(); + let relative = path + .strip_prefix(tree_dir) + .expect("walked path is under the tree") + .to_string_lossy() + .into_owned(); + if is_excluded(&relative, excludes) { + if path.is_dir() { + std::fs::remove_dir_all(&path).with_context(|| { + format!("failed to remove excluded dir {}", path.display()) + })?; + } else { + std::fs::remove_file(&path).with_context(|| { + format!("failed to remove excluded file {}", path.display()) + })?; + } + removed += 1; + } else if path.is_dir() { + stack.push(path); + } + } + } + Ok(removed) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn watch() -> crate::upstream::watch::WatchSource { + crate::upstream::watch::WatchSource::default() + } + + fn package_with_copyright(copyright: &str) -> debmagic_common::package::SourcePackage { + debmagic_common::package::SourcePackage::from_files([ + ( + "changelog", + "pkg (1.0-1) unstable; urgency=medium\n\n * Change.\n\n -- A Mon, 01 Jan 2024 00:00:00 +0000\n", + ), + ("copyright", copyright), + ]) + .unwrap() + } + + #[test] + fn test_load_repack_config_from_copyright() { + let package = package_with_copyright( + "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\nFiles-Excluded:\n win32/*\n docs/html/*\n\nFiles: *\nCopyright: X\nLicense: MIT\n", + ); + + let config = load_repack_config(&package, &watch()).unwrap(); + assert_eq!(config.excludes, vec!["win32/*", "docs/html/*"]); + assert_eq!(config.suffix, None); + } + + #[test] + fn test_load_repack_config_component_field() { + let package = package_with_copyright( + "Files-Excluded-pigeonhole: doc/rfc\n\nFiles: *\nCopyright: X\n", + ); + + let config = load_repack_config(&package, &watch()).unwrap(); + assert_eq!(config.excludes, vec!["doc/rfc"]); + } + + #[test] + fn test_load_repack_config_suffix_from_watch() { + let package = package_with_copyright("Files: *\nCopyright: X\n"); + + let mut source = watch(); + source.repack_suffix = Some("+dfsg1".to_string()); + let config = load_repack_config(&package, &source).unwrap(); + assert_eq!(config.suffix.as_deref(), Some("+dfsg1")); + } + + #[test] + fn test_apply_excludes() { + let dir = std::env::temp_dir().join(format!("debmagic-excl-{}", std::process::id())); + std::fs::create_dir_all(dir.join("win32").join("nested")).unwrap(); + std::fs::create_dir_all(dir.join("src")).unwrap(); + std::fs::write(dir.join("win32").join("a.c"), "x").unwrap(); + std::fs::write(dir.join("win32").join("nested").join("b.c"), "x").unwrap(); + std::fs::write(dir.join("src").join("main.c"), "x").unwrap(); + std::fs::write(dir.join("top.c"), "x").unwrap(); + + let removed = apply_excludes(&dir, &["win32/*".to_string()]).unwrap(); + assert_eq!(removed, 1); + assert!(!dir.join("win32").exists()); + assert!(dir.join("src").join("main.c").exists()); + assert!(dir.join("top.c").exists()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn test_is_excluded() { + let excludes = vec!["win32/*".to_string(), "*.pdf".to_string()]; + assert!(is_excluded("win32/a.c", &excludes)); + assert!(is_excluded("win32/nested/b.c", &excludes)); + assert!(is_excluded("docs/manual.pdf", &excludes)); + assert!(!is_excluded("src/main.c", &excludes)); + assert!(!is_excluded("win32.c", &excludes)); + } +} diff --git a/packages/debmagic/src/upstream/switch.rs b/packages/debmagic/src/upstream/switch.rs new file mode 100644 index 00000000..99d96cfd --- /dev/null +++ b/packages/debmagic/src/upstream/switch.rs @@ -0,0 +1,605 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Context, bail}; + +use crate::upstream::mangle::mangle; +use crate::upstream::orig::{OrigTarballConfig, fetch_orig_tarball_for_version}; +use crate::upstream::query::Candidate; +use crate::upstream::repack::{RepackConfig, apply_excludes}; +use crate::upstream::watch::WatchSource; + +/// Resolve the absolute download URL for a candidate: relative hrefs +/// resolve against the watch source's URL, then `Download-Url-Mangle` +/// applies. +pub fn download_url(source: &WatchSource, candidate: &Candidate) -> anyhow::Result { + let absolute = candidate.href.starts_with("http://") + || candidate.href.starts_with("https://") + || candidate.href.starts_with("ftp://") + || candidate.href.starts_with("ftps://"); + let mut url = if absolute { + candidate.href.clone() + } else { + let base = source.source.trim_end_matches('/'); + format!("{base}/{}", candidate.href.trim_start_matches('/')) + }; + if let Some(rules) = &source.download_url_mangle { + url = mangle(rules, &url)?; + } + Ok(url) +} + +/// The local filename for a downloaded tarball: `Filename-Mangle` if +/// set, else the last URL component without any query fragment. +pub fn download_filename(source: &WatchSource, candidate: &Candidate) -> anyhow::Result { + if let Some(rules) = &source.filename_mangle { + return mangle(rules, &candidate.href); + } + let url = download_url(source, candidate)?; + let last = url.rsplit('/').next().unwrap_or_default(); + Ok(last + .split(['?', '#']) + .next() + .unwrap_or_default() + .to_string()) +} + +/// Download a candidate's tarball into `download_dir`, returning its path. +/// Uses curl for ftp, the requests module for http(s). +pub async fn download_tarball( + source: &WatchSource, + candidate: &Candidate, + download_dir: &Path, +) -> anyhow::Result { + let url = download_url(source, candidate)?; + let filename = download_filename(source, candidate)?; + if filename.is_empty() { + bail!("could not determine a filename for {url}"); + } + let destination = download_dir.join(filename); + if destination.exists() { + println!( + "debmagic: using already-downloaded {}", + destination.display() + ); + return Ok(destination); + } + println!("debmagic: downloading {url}"); + if url.starts_with("ftp://") || url.starts_with("ftps://") { + crate::requests::ftp_download(&url, &destination).await?; + } else { + crate::requests::http_download(&url, &destination).await?; + } + Ok(destination) +} + +/// Download and verify the signature of a downloaded tarball. +/// The signature URL comes from `Pgp-Sig-Url-Mangle` or the common +/// suffixes (`.asc`, `.sig`, ...); verification runs against +/// `debian/upstream/signing-key.asc` when that file exists. +/// Returns Ok(()) when no keyring or no signature is available. +pub async fn verify_tarball_signature( + source: &WatchSource, + source_dir: &Path, + candidate: &Candidate, + tarball: &Path, + sign_options: &crate::sign::SignOptions, +) -> anyhow::Result<()> { + let keyring = source_dir + .join("debian") + .join("upstream") + .join("signing-key.asc"); + if !keyring.is_file() { + return Ok(()); + } + + let url = download_url(source, candidate)?; + let sig_url = match &source.pgp_sig_url_mangle { + Some(rules) => mangle(rules, &url)?, + None => { + let mut found = None; + for suffix in ["asc", "sig", "sign", "pgp", "gpg"] { + let candidate_url = format!("{url}.{suffix}"); + if crate::requests::http_exists(&candidate_url).await { + found = Some(candidate_url); + break; + } + } + match found { + Some(url) => url, + None => return Ok(()), + } + } + }; + + let sig_name = sig_url.rsplit('/').next().unwrap_or_default(); + let sig_path = tarball + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(sig_name); + println!("debmagic: downloading signature {sig_url}"); + crate::requests::http_download(&sig_url, &sig_path).await?; + + crate::sign::verify_signature(sign_options, tarball, &sig_path, &keyring)?; + println!( + "debmagic: verified upstream signature of {}", + tarball.display() + ); + Ok(()) +} + +/// Extract a tarball into `extract_dir`, stripping the leading +/// `-/` component so the tree sits directly in +/// `extract_dir`. +pub fn extract_tarball(tarball: &Path, extract_dir: &Path) -> anyhow::Result<()> { + std::fs::create_dir_all(extract_dir) + .with_context(|| format!("failed to create {}", extract_dir.display()))?; + let output = std::process::Command::new("tar") + .arg("--extract") + .arg("--file") + .arg(tarball) + .arg("--directory") + .arg(extract_dir) + .arg("--strip-components=1") + .output() + .with_context(|| format!("failed to run tar for {}", tarball.display()))?; + if !output.status.success() { + bail!( + "extracting {} failed: {}", + tarball.display(), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(()) +} + +/// The result of a switch: what was done and to which version. +#[derive(Debug)] +pub struct SwitchResult { + pub version: String, + pub orig_tarball: PathBuf, + pub removed_files: usize, +} + +/// Switch the package tree to `candidate`'s upstream version: +pub struct SwitchOptions<'a> { + pub repack: &'a RepackConfig, + pub output_dir: &'a Path, + /// How to fetch an orig tarball the distros may already have; + /// `None` skips the distro lookup. + pub orig_tarball_config: Option<&'a OrigTarballConfig>, + pub verify_signatures: bool, + pub sign_options: &'a crate::sign::SignOptions, + pub dry_run: bool, +} + +/// download, extract, apply the repack excludes, write the orig +/// tarball next to the source tree, and swap the tree contents +/// (keeping `debian/`). With `dry_run`, only report what would happen. +pub async fn switch( + source_dir: &Path, + source: &WatchSource, + candidate: &Candidate, + options: &SwitchOptions<'_>, +) -> anyhow::Result { + let package = crate::package::load_package(source_dir)?; + let name = package.name().to_string(); + let repack = options.repack; + let output_dir = options.output_dir; + let dry_run = options.dry_run; + let version = &candidate.version; + let oversion = match &repack.suffix { + Some(suffix) => format!("{version}{suffix}"), + None => version.clone(), + }; + + let work_dir = + std::env::temp_dir().join(format!("debmagic-switch-{}-{}", name, std::process::id())); + let download_dir = work_dir.join("download"); + let extract_dir = work_dir.join("tree"); + + if dry_run { + let url = download_url(source, candidate)?; + println!("debmagic: would switch {} to upstream {version}", name); + println!("debmagic: tarball: {url}"); + if !repack.excludes.is_empty() { + println!("debmagic: would repack, excluding:"); + for pattern in &repack.excludes { + println!(" {pattern}"); + } + } + if let Some(suffix) = &repack.suffix { + println!("debmagic: version suffix: {suffix}"); + } + println!("debmagic: would replace the source tree, keeping debian/"); + return Ok(SwitchResult { + version: oversion, + orig_tarball: download_dir.join(download_filename(source, candidate)?), + removed_files: 0, + }); + } + + std::fs::create_dir_all(&download_dir) + .with_context(|| format!("failed to create {}", download_dir.display()))?; + + // the distros may already have this version (a re-switch, or a + // version Debian/Ubuntu published): their orig tarball must be + // reused, since a re-created one would have different checksums + // and the upload would be rejected. Only when they don't have it + // is the tarball fetched from upstream and repacked. + let mut distro_orig = None; + if let Some(orig_config) = options.orig_tarball_config { + match fetch_orig_tarball_for_version( + orig_config, + &package, + options.output_dir, + &oversion, + source.component.as_deref(), + ) + .await + { + Ok(Some(tarball)) => { + println!( + "debmagic: using orig tarball {} (already available; \ + skipping the upstream download and repack)", + tarball.display() + ); + distro_orig = Some(tarball); + } + Ok(None) => {} + Err(error) => { + println!( + "debmagic: no orig tarball from the configured method ({error}); \ + fetching from upstream" + ); + } + } + } + + let tarball = match &distro_orig { + Some(tarball) => tarball.clone(), + None => { + let tarball = download_tarball(source, candidate, &download_dir).await?; + + if options.verify_signatures { + verify_tarball_signature( + source, + source_dir, + candidate, + &tarball, + options.sign_options, + ) + .await?; + } + tarball + } + }; + + extract_tarball(&tarball, &extract_dir)?; + let removed = apply_excludes(&extract_dir, &repack.excludes)?; + + // canonical Debian layout: one top-level `-/` dir + let top_level = format!("{}-{}", name, oversion); + let stage_dir = work_dir.join("stage").join(&top_level); + std::fs::create_dir_all(&stage_dir) + .with_context(|| format!("failed to create {}", stage_dir.display()))?; + move_tree_contents(&extract_dir, &stage_dir)?; + + // swap the tree first: it removes stale entries, which must not + // touch the freshly written orig in the output dir + swap_tree(source_dir, &stage_dir)?; + + // an orig tarball for this version may already exist — from a prior + // switch or a developer-provided one. Re-creating it would produce + // different checksums than the archive already holds, and the upload + // would be rejected, so the existing one is kept as-is. + std::fs::create_dir_all(output_dir) + .with_context(|| format!("failed to create {}", output_dir.display()))?; + let orig_path = + match find_existing_orig(output_dir, &name, &oversion, source.component.as_deref()) { + Some(existing) => { + println!( + "debmagic: keeping existing orig tarball {} (re-creating it would \ + change its checksums and break uploads)", + existing.display() + ); + existing + } + None => { + let orig_name = match &source.component { + Some(component) => format!( + "{}xz", + debmagic_common::changes::component_orig_prefix( + &name, &oversion, component + ) + ), + None => format!( + "{}xz", + debmagic_common::changes::orig_prefix(&name, &oversion) + ), + }; + let orig_path = output_dir.join(&orig_name); + let output = std::process::Command::new("tar") + .current_dir(stage_dir.parent().expect("stage dir has a parent")) + .args(["--create", "--xz", "--file"]) + .arg(&orig_path) + .arg(&top_level) + .output() + .with_context(|| format!("failed to run tar for {}", orig_path.display()))?; + if !output.status.success() { + bail!( + "creating {} failed: {}", + orig_path.display(), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + orig_path + } + }; + + std::fs::remove_dir_all(&work_dir).ok(); + println!( + "debmagic: switched {} to upstream {oversion} (orig: {}, {} files excluded)", + name, + orig_path.display(), + removed + ); + Ok(SwitchResult { + version: oversion, + orig_tarball: orig_path, + removed_files: removed, + }) +} + +/// An orig tarball for this version that already exists in the output +/// dir — from a prior switch or a developer-provided one. Re-creating +/// it would produce different checksums than the archive already +/// holds, and the upload would be rejected, so it is kept as-is. +fn find_existing_orig( + output_dir: &Path, + name: &str, + oversion: &str, + component: Option<&str>, +) -> Option { + match component { + Some(component) => { + debmagic_common::changes::find_component_in_dir(output_dir, name, oversion, component) + } + None => debmagic_common::changes::find_orig_in_dir(output_dir, name, oversion), + } +} + +/// Move every entry of `from_dir` into `to_dir`. +fn move_tree_contents(from_dir: &Path, to_dir: &Path) -> anyhow::Result<()> { + for entry in std::fs::read_dir(from_dir) + .with_context(|| format!("failed to read {}", from_dir.display()))? + .flatten() + { + let destination = to_dir.join(entry.file_name()); + move_entry(&entry.path(), &destination)?; + } + Ok(()) +} + +/// Move one entry, falling back to copy+remove when a rename crosses +/// filesystems (the work dir is in /tmp, the source tree may not be). +fn move_entry(from: &Path, to: &Path) -> anyhow::Result<()> { + if std::fs::rename(from, to).is_ok() { + return Ok(()); + } + if from.is_dir() { + copy_dir(from, to)?; + std::fs::remove_dir_all(from) + } else { + std::fs::copy(from, to) + .map(|_| ()) + .and_then(|()| std::fs::remove_file(from)) + } + .with_context(|| format!("failed to move {} to {}", from.display(), to.display())) +} + +/// Recursively copy a directory. Symlinks are recreated as symlinks — +/// a tarball may contain dangling ones, and following them would fail. +fn copy_dir(from: &Path, to: &Path) -> anyhow::Result<()> { + std::fs::create_dir_all(to).with_context(|| format!("failed to create {}", to.display()))?; + for entry in std::fs::read_dir(from) + .with_context(|| format!("failed to read {}", from.display()))? + .flatten() + { + let destination = to.join(entry.file_name()); + let path = entry.path(); + if entry.file_type()?.is_symlink() { + let target = std::fs::read_link(&path) + .with_context(|| format!("failed to read link {}", path.display()))?; + // a leftover entry from an interrupted run must not block the swap + if std::fs::symlink_metadata(&destination).is_ok() { + if destination.is_dir() { + std::fs::remove_dir_all(&destination) + .with_context(|| format!("failed to remove {}", destination.display()))?; + } else { + std::fs::remove_file(&destination) + .with_context(|| format!("failed to remove {}", destination.display()))?; + } + } + std::os::unix::fs::symlink(&target, &destination).with_context(|| { + format!( + "failed to create symlink {} -> {}", + destination.display(), + target.display() + ) + })?; + } else if path.is_dir() { + copy_dir(&path, &destination)?; + } else { + std::fs::copy(&path, &destination).with_context(|| { + format!( + "failed to copy {} to {}", + path.display(), + destination.display() + ) + })?; + } + } + Ok(()) +} + +/// Replace the source tree's contents with the extracted tree's, +/// keeping `debian/` and any VCS metadata dir (`.git`, ...) intact. +fn swap_tree(source_dir: &Path, extract_dir: &Path) -> anyhow::Result<()> { + let new_names: Vec = std::fs::read_dir(extract_dir) + .with_context(|| format!("failed to read {}", extract_dir.display()))? + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + + // stale entries from the old tree that the new one doesn't have + for entry in std::fs::read_dir(source_dir) + .with_context(|| format!("failed to read {}", source_dir.display()))? + .flatten() + { + let name = entry.file_name().to_string_lossy().into_owned(); + if is_kept(&name) || new_names.contains(&name) { + continue; + } + let path = entry.path(); + if path.is_dir() { + std::fs::remove_dir_all(&path) + .with_context(|| format!("failed to remove {}", path.display()))?; + } else { + std::fs::remove_file(&path) + .with_context(|| format!("failed to remove {}", path.display()))?; + } + } + + // move the new tree in + for name in &new_names { + if is_kept(name) { + continue; + } + move_entry(&extract_dir.join(name), &source_dir.join(name))?; + } + Ok(()) +} + +/// Entries that always survive a tree swap: the packaging dir and +/// VCS storage (`.git`) +fn is_kept(name: &str) -> bool { + name == "debian" || name == ".git" +} + +#[cfg(test)] +mod tests { + use super::*; + + fn source() -> WatchSource { + WatchSource { + source: "https://example.com/releases/".to_string(), + matching_pattern: "foo-@ANY_VERSION@@ARCHIVE_EXT@".to_string(), + ..Default::default() + } + } + + #[test] + fn test_download_url_relative() { + let source = source(); + let candidate = Candidate { + version: "1.0".to_string(), + href: "foo-1.0.tar.gz".to_string(), + }; + assert_eq!( + download_url(&source, &candidate).unwrap(), + "https://example.com/releases/foo-1.0.tar.gz" + ); + } + + #[test] + fn test_download_url_absolute() { + let source = source(); + let candidate = Candidate { + version: "1.0".to_string(), + href: "https://mirror.example.com/foo-1.0.tar.gz".to_string(), + }; + assert_eq!( + download_url(&source, &candidate).unwrap(), + "https://mirror.example.com/foo-1.0.tar.gz" + ); + } + + #[test] + fn test_download_url_mangle() { + let mut source = source(); + source.download_url_mangle = Some("s/prdownload/download/".to_string()); + let candidate = Candidate { + version: "1.0".to_string(), + href: "https://prdownload.example.com/foo-1.0.tar.gz".to_string(), + }; + assert_eq!( + download_url(&source, &candidate).unwrap(), + "https://download.example.com/foo-1.0.tar.gz" + ); + } + + #[test] + fn test_download_filename() { + let source = source(); + let candidate = Candidate { + version: "1.0".to_string(), + href: "foo-1.0.tar.gz".to_string(), + }; + assert_eq!( + download_filename(&source, &candidate).unwrap(), + "foo-1.0.tar.gz" + ); + } + + #[test] + fn test_find_existing_orig() { + let dir = std::env::temp_dir().join(format!("debmagic-switch-orig-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + + // nothing there yet + assert_eq!(find_existing_orig(&dir, "pkg", "1.0", None), None); + + // a main orig for the version is found, any compression + std::fs::write(dir.join("pkg_1.0.orig.tar.xz"), "x").unwrap(); + assert_eq!( + find_existing_orig(&dir, "pkg", "1.0", None), + Some(dir.join("pkg_1.0.orig.tar.xz")) + ); + // a different version is not + assert_eq!(find_existing_orig(&dir, "pkg", "1.1", None), None); + + // component origs are only found for their component + std::fs::write(dir.join("pkg_1.0.orig-bar.tar.xz"), "x").unwrap(); + assert_eq!( + find_existing_orig(&dir, "pkg", "1.0", Some("bar")), + Some(dir.join("pkg_1.0.orig-bar.tar.xz")) + ); + assert_eq!(find_existing_orig(&dir, "pkg", "1.0", Some("baz")), None); + + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn test_extract_and_swap() { + let base = std::env::temp_dir().join(format!("debmagic-switch-{}", std::process::id())); + let source_dir = base.join("pkg"); + let extract_dir = base.join("tree"); + std::fs::create_dir_all(source_dir.join("debian")).unwrap(); + std::fs::create_dir_all(&extract_dir).unwrap(); + // old tree: a file that the new tree lacks, plus debian/ + std::fs::write(source_dir.join("old.txt"), "old").unwrap(); + std::fs::write(source_dir.join("debian").join("changelog"), "keep").unwrap(); + // new tree + std::fs::write(extract_dir.join("new.txt"), "new").unwrap(); + std::fs::create_dir_all(extract_dir.join("src")).unwrap(); + std::fs::write(extract_dir.join("src").join("main.c"), "x").unwrap(); + + swap_tree(&source_dir, &extract_dir).unwrap(); + + assert!(source_dir.join("new.txt").exists()); + assert!(source_dir.join("src").join("main.c").exists()); + assert!(!source_dir.join("old.txt").exists()); + assert!(source_dir.join("debian").join("changelog").exists()); + std::fs::remove_dir_all(&base).unwrap(); + } +} diff --git a/packages/debmagic/src/upstream/watch.rs b/packages/debmagic/src/upstream/watch.rs new file mode 100644 index 00000000..a1af5d30 --- /dev/null +++ b/packages/debmagic/src/upstream/watch.rs @@ -0,0 +1,432 @@ +use std::path::Path; + +use anyhow::{Context, bail}; + +/// A parsed `debian/watch` source: one paragraph (v5) or line (v2–v4). +#[derive(Debug, Clone, Default)] +pub struct WatchSource { + pub source: String, + pub matching_pattern: String, + pub search_mode: SearchMode, + pub uversion_mangle: Option, + pub dversion_mangle: Option, + pub filename_mangle: Option, + pub download_url_mangle: Option, + pub pgp_sig_url_mangle: Option, + pub pgp_mode: Option, + pub repack_suffix: Option, + pub repack: bool, + /// MUT component name: this source describes one component tarball + /// of a multiple-upstream-tarballs package. + pub component: Option, + pub untrackable: Option, +} + +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq)] +pub enum SearchMode { + #[default] + Html, + Plain, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum PgpMode { + Auto, + Default, + Mangle, + None, +} + +/// Parse `debian/watch` (versions 2–5). Version 1 is rejected; +/// unsupported options produce a clear error. +pub fn parse_watch_file(path: &Path) -> anyhow::Result> { + let content = std::fs::read_to_string(path) + .with_context(|| format!("failed to read {}", path.display()))?; + parse_watch(&content) +} + +/// Parse a watch file's contents. The first non-comment line determines +/// the version: `version=2|3|4` (single-line format) or `Version: 5` +/// (deb822 paragraphs). +pub fn parse_watch(content: &str) -> anyhow::Result> { + let version = detect_version(content)?; + match version { + 2..=4 => parse_watch_v4(content), + 5 => parse_watch_v5(content), + _ => bail!("unsupported watch file version {version}"), + } +} + +/// The version declared by the file, from `version=N` or `Version: N`. +fn detect_version(content: &str) -> anyhow::Result { + for line in content.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + if let Some(rest) = line.strip_prefix("version=") { + return rest + .trim() + .parse() + .with_context(|| format!("invalid watch version: {rest}")); + } + if let Some(rest) = line.strip_prefix("Version:") { + return rest + .trim() + .parse() + .with_context(|| format!("invalid watch version: {rest}")); + } + bail!( + "watch file has no version declaration; add 'Version: 5' (version 1 files are not supported)" + ); + } + bail!("watch file is empty"); +} + +/// v2–v4: one `opts=...` line per source, with backslash continuations joined. +fn parse_watch_v4(content: &str) -> anyhow::Result> { + let mut sources = Vec::new(); + let mut logical = String::new(); + for line in content.lines() { + let line = line.trim_end(); + if let Some(cont) = line.strip_suffix('\\') { + logical.push_str(cont.trim_end()); + logical.push(' '); + continue; + } + logical.push_str(line); + let joined = logical.trim().to_string(); + logical.clear(); + if joined.is_empty() || joined.starts_with('#') || joined.starts_with("version=") { + continue; + } + sources.push(parse_watch_line(&joined)?); + } + if sources.is_empty() { + bail!("watch file declares no sources"); + } + Ok(sources) +} + +/// One v2–v4 line: `[opts=...] [matching-pattern]`. The pattern may +/// be a separate token or embedded in the URL as its last `/`-separated +/// component (uscan's shorthand). +fn parse_watch_line(line: &str) -> anyhow::Result { + let mut source = WatchSource::default(); + let mut rest = line; + if let Some(opts) = line.strip_prefix("opts=") { + let (opts, remainder) = opts + .split_once(' ') + .context("opts= without url in watch line")?; + apply_opts(&mut source, opts)?; + rest = remainder.trim_start(); + } + match rest.split_once(' ') { + Some((url, pattern)) => { + source.source = url.to_string(); + source.matching_pattern = pattern.to_string(); + } + // shorthand: the pattern is the URL's last `/` component + None => { + let (url, pattern) = rest + .rsplit_once('/') + .with_context(|| format!("watch line has no matching pattern: {line}"))?; + source.source = format!("{url}/"); + source.matching_pattern = pattern.to_string(); + } + } + Ok(source) +} + +/// v5: deb822 paragraphs; the first paragraph's options are defaults +/// for the following source paragraphs. +fn parse_watch_v5(content: &str) -> anyhow::Result> { + let mut paragraphs: Vec> = Vec::new(); + let mut current: Vec<(String, String)> = Vec::new(); + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() { + if !current.is_empty() { + paragraphs.push(std::mem::take(&mut current)); + } + continue; + } + if trimmed.starts_with('#') { + continue; + } + let (key, value) = trimmed + .split_once(':') + .with_context(|| format!("invalid line in watch file: {line}"))?; + current.push((key.trim().to_lowercase(), value.trim().to_string())); + } + if !current.is_empty() { + paragraphs.push(current); + } + + let mut sources = Vec::new(); + // first paragraph holds defaults (and may itself be a source) + let mut defaults = WatchSource::default(); + let mut defaults_are_source = false; + for (index, paragraph) in paragraphs.iter().enumerate() { + let mut source = WatchSource::default(); + for (key, value) in paragraph { + match key.as_str() { + "version" => {} + "untrackable" => source.untrackable = Some(value.clone()), + "source" => { + source.source = value.clone(); + defaults_are_source = index == 0; + } + "matching-pattern" | "matchingpattern" => source.matching_pattern = value.clone(), + "search-mode" | "searchmode" => { + source.search_mode = match value.to_lowercase().as_str() { + "html" => SearchMode::Html, + "plain" => SearchMode::Plain, + _ => bail!("invalid Search-Mode: {value}"), + } + } + "uversion-mangle" | "uversionmangle" => { + source.uversion_mangle = Some(value.clone()) + } + "dversion-mangle" | "dversionmangle" => { + source.dversion_mangle = Some(value.clone()) + } + "filename-mangle" | "filenamemangle" => { + source.filename_mangle = Some(value.clone()) + } + "download-url-mangle" | "downloadurlmangle" => { + source.download_url_mangle = Some(value.clone()) + } + "pgp-sig-url-mangle" | "pgpsigurlmangle" => { + source.pgp_sig_url_mangle = Some(value.clone()) + } + "pgp-mode" | "pgpmode" => { + source.pgp_mode = Some(match value.to_lowercase().as_str() { + "auto" => PgpMode::Auto, + "default" => PgpMode::Default, + "mangle" => PgpMode::Mangle, + "none" => PgpMode::None, + other => bail!("unsupported Pgp-Mode: {other}"), + }) + } + "repack" => source.repack = value.eq_ignore_ascii_case("yes"), + "repacksuffix" => source.repack_suffix = Some(value.clone()), + "component" => source.component = Some(value.clone()), + "ctype" | "version-schema" | "versionseparator" | "update-script" + | "page-mangle" | "mode" | "compression" | "unzip-opt" | "git-pretty" + | "git-date" | "git-export" | "git-mode" | "git-modules" | "bare" | "maturity" + | "hrefdecode" | "decompress" | "user-agent" | "template" | "dist" | "owner" + | "project" | "oversion-mangle" => { + bail!( + "watch file option '{key}' is not supported yet; \ + declare the upstream source in debmagic.toml instead" + ) + } + _ => bail!("unknown watch file option: {key}"), + } + } + if index == 0 { + // first paragraph: defaults, possibly also a source itself + defaults = source.clone(); + if defaults_are_source { + sources.push(source); + } + } else { + // merge over the first paragraph's defaults + let merged = merge_source(&defaults, source); + sources.push(merged); + } + } + if sources.is_empty() { + bail!("watch file declares no sources"); + } + Ok(sources) +} + +/// Field-by-field merge of a source over defaults, `None`/empty fields +/// falling back to the defaults. +fn merge_source(defaults: &WatchSource, source: WatchSource) -> WatchSource { + let mut merged = source; + if merged.source.is_empty() { + merged.source = defaults.source.clone(); + } + if merged.matching_pattern.is_empty() { + merged.matching_pattern = defaults.matching_pattern.clone(); + } + if merged.search_mode == defaults.search_mode { + // nothing to do; both default or both overridden identically + } + merged.uversion_mangle = merged + .uversion_mangle + .or_else(|| defaults.uversion_mangle.clone()); + merged.dversion_mangle = merged + .dversion_mangle + .or_else(|| defaults.dversion_mangle.clone()); + merged.filename_mangle = merged + .filename_mangle + .or_else(|| defaults.filename_mangle.clone()); + merged.download_url_mangle = merged + .download_url_mangle + .or_else(|| defaults.download_url_mangle.clone()); + merged.pgp_sig_url_mangle = merged + .pgp_sig_url_mangle + .or_else(|| defaults.pgp_sig_url_mangle.clone()); + merged.pgp_mode = merged.pgp_mode.or(defaults.pgp_mode); + merged.repack_suffix = merged + .repack_suffix + .or_else(|| defaults.repack_suffix.clone()); + merged.repack = merged.repack || defaults.repack; + merged.component = merged.component.or_else(|| defaults.component.clone()); + merged +} + +/// Apply `opts=` content of a v2–v4 line. +fn apply_opts(source: &mut WatchSource, opts: &str) -> anyhow::Result<()> { + for opt in opts.split(',') { + let opt = opt.trim(); + if let Some(value) = opt.strip_prefix("uversionmangle=") { + source.uversion_mangle = Some(value.to_string()); + } else if let Some(value) = opt.strip_prefix("dversionmangle=") { + source.dversion_mangle = Some(value.to_string()); + } else if let Some(value) = opt.strip_prefix("filenamemangle=") { + source.filename_mangle = Some(value.to_string()); + } else if let Some(value) = opt.strip_prefix("downloadurlmangle=") { + source.download_url_mangle = Some(value.to_string()); + } else if let Some(value) = opt.strip_prefix("pgpsigurlmangle=") { + source.pgp_sig_url_mangle = Some(value.to_string()); + } else if let Some(value) = opt.strip_prefix("pgpmode=") { + source.pgp_mode = Some(match value { + "auto" => PgpMode::Auto, + "default" => PgpMode::Default, + "mangle" => PgpMode::Mangle, + "none" => PgpMode::None, + other => bail!("unsupported pgpmode: {other}"), + }); + } else if let Some(value) = opt.strip_prefix("repacksuffix=") { + source.repack_suffix = Some(value.to_string()); + } else if opt == "repack" { + source.repack = true; + } else if opt == "pasv" || opt == "passive" { + // FTP-only option, irrelevant for our fetcher + } else if let Some(value) = opt.strip_prefix("searchmode=") { + source.search_mode = match value { + "html" => SearchMode::Html, + "plain" => SearchMode::Plain, + _ => bail!("invalid searchmode: {value}"), + }; + } else { + bail!( + "watch file option '{opt}' is not supported yet; \ + declare the upstream source in debmagic.toml instead" + ); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_case::test_case; + + #[test] + fn test_parse_v4_basic() { + let sources = + parse_watch("version=4\nhttps://example.com/releases/ foo-(.*)\\.tar\\.gz\n").unwrap(); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].source, "https://example.com/releases/"); + assert_eq!(sources[0].matching_pattern, "foo-(.*)\\.tar\\.gz"); + } + + #[test] + fn test_parse_v4_opts() { + let sources = parse_watch( + "version=4\n\ + opts=uversionmangle=s/(\\d)[_\\.\\-\\+]?((?:RC|rc|pre|dev|beta|alpha)\\d*)$/$1~$2/ \\\n\ + https://example.com/ foo-(.*)\\.tar\\.gz\n", + ) + .unwrap(); + assert!(sources[0].uversion_mangle.is_some()); + } + + #[test] + fn test_parse_v5_basic() { + let sources = parse_watch( + "Version: 5\nSource: https://example.com/releases/\nMatching-Pattern: foo-@ANY_VERSION@@ARCHIVE_EXT@\n", + ) + .unwrap(); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].source, "https://example.com/releases/"); + assert_eq!( + sources[0].matching_pattern, + "foo-@ANY_VERSION@@ARCHIVE_EXT@" + ); + } + + #[test] + fn test_parse_v5_defaults_apply() { + let sources = parse_watch( + "Version: 5\n\ + Uversion-Mangle: s/RC/~rc/\n\ + Source: https://example.com/a/\nMatching-Pattern: a-@ANY_VERSION@\n\ + \n\ + Source: https://example.com/b/\nMatching-Pattern: b-@ANY_VERSION@\n", + ) + .unwrap(); + assert_eq!(sources.len(), 2); + assert_eq!(sources[1].uversion_mangle.as_deref(), Some("s/RC/~rc/")); + } + + #[test] + fn test_parse_v5_untrackable() { + let sources = + parse_watch("Version: 5\nUntrackable: on hold\nSource: https://example.com/\n") + .unwrap(); + assert_eq!(sources[0].untrackable.as_deref(), Some("on hold")); + } + + #[test] + fn test_parse_v5_dfsg() { + let sources = parse_watch( + "Version: 5\n\ + Source: https://example.com/\n\ + Matching-Pattern: foo-@ANY_VERSION@@ARCHIVE_EXT@\n\ + Dversion-Mangle: s/\\+dfsg\\d*$//\n\ + Repacksuffix: +dfsg\n", + ) + .unwrap(); + assert_eq!(sources[0].repack_suffix.as_deref(), Some("+dfsg")); + assert_eq!( + sources[0].dversion_mangle.as_deref(), + Some("s/\\+dfsg\\d*$//") + ); + } + + #[test_case("version=1\nhttps://example.com/ foo\n"; "v1 rejected")] + #[test_case("https://example.com/ foo\n"; "no version")] + #[test_case( + "Version: 5\nSource: https://example.com/\nMode: git\n"; + "unsupported mode" + )] + #[test_case( + "Version: 5\nSource: https://example.com/\nVersion-Schema: group\n"; + "unsupported version schema" + )] + fn test_parse_rejects(content: &str) { + assert!(parse_watch(content).is_err()); + } + + #[test] + fn test_parse_v5_component() { + let sources = parse_watch( + "Version: 5\n\ + Source: https://example.com/a/\nMatching-Pattern: a-@ANY_VERSION@\n\ + \n\ + Source: https://example.com/b/\nMatching-Pattern: b-@ANY_VERSION@\nComponent: bar\n", + ) + .unwrap(); + assert_eq!(sources.len(), 2); + assert_eq!(sources[0].component, None); + assert_eq!(sources[1].component.as_deref(), Some("bar")); + } +} diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml new file mode 100644 index 00000000..ec68826b --- /dev/null +++ b/snap/snapcraft.yaml @@ -0,0 +1,34 @@ +name: debmagic +title: debmagic +summary: Build and package Debian/Ubuntu packages in isolated environments +description: | + Modern, robust & easy [Debian](https://debian.org)/[Ubuntu](https://ubuntu.com) packaging - while staying backwards compatible. + + Debmagic includes tools for the whole packaging workflow: building, testing, signing, ... + It operates in isolated containers, and supports the developer with accessible introspection, debugging and linting. +license: GPL-2.0-or-later +source-code: https://github.com/SFTtech/debmagic +website: https://debmagic.readthedocs.io +contact: https://github.com/SFTtech/debmagic/issues + +type: app +base: core26 +adopt-info: debmagic +grade: devel +confinement: strict + +apps: + debmagic: + command: bin/debmagic + +parts: + debmagic: + plugin: rust + source: . + rust-path: [packages/debmagic] + rust-channel: "none" + build-packages: [git, rustc, cargo] + override-pull: | + craftctl default + # store has 32 char limit (-> use +d as dirty and format manually). + craftctl set version=$(git describe --tags --always --dirty=+d | sed 's/^v//; s/-\([0-9]*\)-g/.\1+git/')