Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 0 additions & 9 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,6 @@ concurrency:
group: ci-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

# One run per pull request at a time. Pushing again while checks are running
# supersedes them, and without this the old run keeps a runner busy: a handful
# of quick force-pushes left a dozen doomed runs queued ahead of the one that
# mattered, and everything behind them starved. Pushes to main are left alone —
# each commit there deserves its own verdict.
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

# Force JavaScript-based actions to run on Node 24 instead of the deprecated Node 20 ahead of
# the 2026-06-16 hard cutover, until we bump each action to a Node-24-native major version.
env:
Expand Down
68 changes: 68 additions & 0 deletions Compression.Lib/FormatDetector.cs
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,23 @@ public static Format DetectByExtension(string path) {
return cpc;
}

// ".pak" is Quake's PACK archive and Unreal's package. Quake's magic is the
// leading "PACK"; Unreal's sits in the footer, so a leading-bytes check can
// never see it and every Unreal pak was routed to the Quake reader.
if (singleExt == ".pak") {
var pak = DetectPakByMagic(path);
if (pak != Format.Unknown)
return pak;
}

// ".vib" is both a VMware installation bundle (an AR archive) and a Veeam
// incremental backup. Only one of the two says what it is up front.
if (singleExt == ".vib") {
var vib = DetectVibByMagic(path);
if (vib != Format.Unknown)
return vib;
}

// The ".arc" extension is shared by the legacy SEA ARC format (every entry
// header starts with the 0x1A magic byte, the "Arc" descriptor) and FreeArc
// (magic "ArC\x01", the "FreeArc" descriptor). The first-claim-wins map routes
Expand Down Expand Up @@ -305,6 +322,57 @@ private static Format DetectCpcDskByMagic(string path) {
/// format by reading the leading bytes. Returns <see cref="Format.Unknown"/>
/// when unreadable so the caller falls back to the registry extension map.
/// </summary>
/// <summary>
/// Tells Quake's PACK archive from an Unreal package. Quake announces itself in
/// the first four bytes; Unreal keeps its magic in the footer, ahead of the
/// trailing index offset and length, so the tail is where it has to be read.
/// </summary>
private static Format DetectPakByMagic(string path) {
try {
if (!File.Exists(path)) return Format.Unknown;
using var fs = File.OpenRead(path);
Span<byte> magic = stackalloc byte[4];
if (fs.Length >= 4) {
fs.ReadExactly(magic);
if (magic[0] == 'P' && magic[1] == 'A' && magic[2] == 'C' && magic[3] == 'K')
return Format.Pak;
}

// The footer is 44 bytes for the versions that keep the magic at its front;
// later revisions prepend fields, so scan the tail rather than fix an offset.
var tail = (int)Math.Min(fs.Length, 256);
if (tail < 4) return Format.Unknown;
var buffer = new byte[tail];
fs.Position = fs.Length - tail;
fs.ReadExactly(buffer);
for (var i = 0; i + 4 <= tail; ++i)
if (BinaryPrimitives.ReadUInt32LittleEndian(buffer.AsSpan(i, 4)) == 0x5A6F12E1)
return Format.UnrealPak;
} catch {
/* ignore detection failure */
}
return Format.Unknown;
}

/// <summary>
/// Tells a VMware installation bundle from a Veeam incremental backup. The
/// bundle is an AR archive and says so in its first eight bytes.
/// </summary>
private static Format DetectVibByMagic(string path) {
try {
if (!File.Exists(path)) return Format.Unknown;
using var fs = File.OpenRead(path);
if (fs.Length < 8) return Format.Unknown;
Span<byte> magic = stackalloc byte[8];
fs.ReadExactly(magic);
if (magic.SequenceEqual("!<arch>\n"u8))
return Format.Vib;
} catch {
/* ignore detection failure */
}
return Format.Unknown;
}

private static Format DetectArcByMagic(string path) {
try {
if (!File.Exists(path)) return Format.Unknown;
Expand Down
19 changes: 19 additions & 0 deletions Compression.Tests/Balz/BalzTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,25 @@ public void RoundTrip(int size) {
Assert.That(decompressed.ToArray(), Is.EqualTo(data));
}

/// <summary>
/// This payload used to desynchronize the coder: on a small enough range the
/// scaled probability truncates to zero and the split point landed one below
/// low, so a zero bit set high under low. One literal came back wrong 43020
/// symbols in, and the next match pointed at an empty slot.
/// </summary>
[Test]
public void RoundTrip_PayloadThatCollapsedTheRange() {
var data = new byte[65536];
new Random(3138).NextBytes(data);
using var input = new MemoryStream(data);
using var compressed = new MemoryStream();
BalzStream.Compress(input, compressed);
compressed.Position = 0;
using var decompressed = new MemoryStream();
BalzStream.Decompress(compressed, decompressed);
Assert.That(decompressed.ToArray(), Is.EqualTo(data));
}

[Test, Category("EdgeCase")]
public void RoundTrip_Empty() {
var data = Array.Empty<byte>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ public sealed class SupportedHashSizesContractTests {
[typeof(Jh)] = static () => Jh.SupportedHashSizes,
[typeof(Sha3)] = static () => Sha3.SupportedHashSizes,
[typeof(KnotHash)] = static () => KnotHash.SupportedHashSizes,
[typeof(Keccak)] = static () => Keccak.SupportedHashSizes,
[typeof(Kupyna)] = static () => Kupyna.SupportedHashSizes,
[typeof(Md2)] = static () => Md2.SupportedHashSizes,
[typeof(Md4)] = static () => Md4.SupportedHashSizes,
Expand Down
2 changes: 2 additions & 0 deletions CompressionWorkbench.slnx
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
<Solution>
<Folder Name="/Core/">
<Project Path="Hawkynt.Algorithms.Hashing/Hawkynt.Algorithms.Hashing.csproj" />
<Project Path="Hawkynt.Algorithms.Checksums/Hawkynt.Algorithms.Checksums.csproj" />
<Project Path="Compression.Core/Hawkynt.Compression.Core.csproj" />
<Project Path="Compression.Registry/Compression.Registry.csproj" />
<Project Path="Compression.Registry.Generator/Compression.Registry.Generator.csproj" />
Expand Down
10 changes: 10 additions & 0 deletions FileFormats/FileFormat.Balz/BalzStream.cs
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,11 @@ private sealed class ArithEncoder {
public void EncodeBit(int bit, ref int prob) {
var range = _high - _low + 1;
var mid = _low + (ulong)range * (uint)prob / (uint)ProbMax - 1;
// The split has to stay inside [low, high). Normalization only guarantees
// the top bytes differ, so the range can be small enough that the scaled
// probability truncates to zero — and then the "- 1" puts mid below low,
// where encoding a zero bit sets high under low and inverts the interval.
if (mid < _low) mid = _low;
if (mid >= _high) mid = _high - 1;
var umid = (uint)mid;

Expand Down Expand Up @@ -235,6 +240,11 @@ public ArithDecoder(Stream input) {
public int DecodeBit(ref int prob) {
var range = _high - _low + 1;
var mid = _low + (ulong)range * (uint)prob / (uint)ProbMax - 1;
// The split has to stay inside [low, high). Normalization only guarantees
// the top bytes differ, so the range can be small enough that the scaled
// probability truncates to zero — and then the "- 1" puts mid below low,
// where encoding a zero bit sets high under low and inverts the interval.
if (mid < _low) mid = _low;
if (mid >= _high) mid = _high - 1;
var umid = (uint)mid;

Expand Down
2 changes: 2 additions & 0 deletions Hawkynt.Algorithms.Checksums/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ Legacy CompressionWorkbench call sites may continue to use the compatibility typ

<!-- API:BEGIN generated by Hawkynt/RepositoryTemplate/package-readme — edit the XML docs in source, not here -->

Every public and protected member of all 69 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Hawkynt.Algorithms.Checksums/REFERENCE.md).

<!-- API:END -->

## 🏗 Architecture
Expand Down
Loading
Loading