diff --git a/.gitignore b/.gitignore index 7bd006b..a8bc02a 100644 --- a/.gitignore +++ b/.gitignore @@ -88,4 +88,5 @@ composer.lock .php-cs-fixer.cache .idea/ -.phpunit.cache \ No newline at end of file +.phpunit.cache +/.phpunit.result.cache diff --git a/src/HyperLogLog.php b/src/HyperLogLog.php index eb0ca5e..50442f3 100644 --- a/src/HyperLogLog.php +++ b/src/HyperLogLog.php @@ -15,7 +15,8 @@ */ class HyperLogLog { - private int $_precalculatedTwoPow32 = 4_294_967_296; + /** @var int */ + private const TWO_POW_32 = 4_294_967_296; private int $counterBits; private string $hashAlgorithm; @@ -25,21 +26,31 @@ class HyperLogLog /** @var array Counters storing maximum rho values */ private array $counters; + /** @var bool Indicates if the selected hash algorithm produces less than 64 bits */ + private bool $needsPadding; + /** * HyperLogLog constructor. * * @param int $counterBits Number of bits used to define the number of counters (m = 2^counterBits). * Higher values improve accuracy but increase memory usage. - * @param string $hashAlgorithm Hash algorithm used for input hashing (e.g. xxh3, murmur3f, sha256). + * @param string $hashAlgorithm Hash algorithm used for input hashing (e.g. xxh3, murmur3f, crc32, sha256). */ public function __construct(int $counterBits = 5, string $hashAlgorithm = 'xxh3') { + if (!in_array($hashAlgorithm, hash_algos())) { + throw new \InvalidArgumentException('Invalid hash algorithm'); + } + $this->counterBits = $counterBits; $this->hashAlgorithm = $hashAlgorithm; $this->m = 1 << $this->counterBits; $this->counters = array_fill(0, $this->m, 0); + + $testHash = hash($this->hashAlgorithm, 'test', true); + $this->needsPadding = strlen($testHash) < 8; } public function getCounterBits(): int @@ -47,37 +58,16 @@ public function getCounterBits(): int return $this->counterBits; } - public function setCounterBits(int $counterBits): HyperLogLog - { - $this->counterBits = $counterBits; - - return $this; - } - public function getHashAlgorithm(): string { return $this->hashAlgorithm; } - public function setHashAlgorithm(string $hashAlgorithm): HyperLogLog - { - $this->hashAlgorithm = $hashAlgorithm; - - return $this; - } - public function getM(): int { return $this->m; } - public function setM(int $m): HyperLogLog - { - $this->m = $m; - - return $this; - } - /** * @return array */ @@ -91,7 +81,7 @@ public function getCounters(): array * * @return $this */ - public function setCounters(array $counters): HyperLogLog + public function setCounters(array $counters): self { $this->counters = $counters; @@ -101,18 +91,34 @@ public function setCounters(array $counters): HyperLogLog /** * Add an element to the HyperLogLog structure. * - * The value is hashed, split into register index and leading zero count, - * and the register is updated with the maximum observed rho value. + * The value is hashed, converted to a 64-bit integer, split into register index + * and leading zero count, and the register is updated with the maximum observed rho value. * * @param string $value element to insert */ public function add(string $value): void { - $hash = $this->hash($value, $this->hashAlgorithm); + $hashString = $this->hash($value, $this->hashAlgorithm); + + // Comblement avec des octets nuls si le hachage fait moins de 64 bits (ex: crc32) + if ($this->needsPadding) { + $hashString = str_pad($hashString, 8, "\x00", STR_PAD_RIGHT); + } + + // Extraction des 8 premiers octets (64 bits) sous forme d'entier (Big Endian) + // unpack('J') ignore nativement tout ce qui dépasse 8 octets (ex: sha256, md5) + $hashIntUnpacked = unpack('J', $hashString); - $counter = $this->counter($hash, $this->counterBits); + if (false === $hashIntUnpacked) { + throw new \InvalidArgumentException('Invalid hash value'); + } + + /** @var int $hashInt */ + $hashInt = $hashIntUnpacked[1]; - $rho = $this->rho($hash, $this->counterBits); + $counter = $this->counter($hashInt, $this->counterBits); + + $rho = $this->rho($hashInt, $this->counterBits); if ($rho > $this->counters[$counter]) { $this->counters[$counter] = $rho; @@ -150,7 +156,7 @@ public function count(): float } // Large range correction - if ($E > $this->_precalculatedTwoPow32 / 30) { + if ($E > $this::TWO_POW_32 / 30) { $E = $this->estimateUsingLargeCardinalitiesApproach($E); } @@ -260,7 +266,7 @@ public function estimateUsingSmallCardinalitiesApproach(int $V, int $m): float */ public function estimateUsingLargeCardinalitiesApproach(float $E): float { - return -$this->_precalculatedTwoPow32 * log(1 - $E / $this->_precalculatedTwoPow32); + return -$this::TWO_POW_32 * log(1 - $E / $this::TWO_POW_32); } /** @@ -277,49 +283,40 @@ private function hash(string $value, string $hashAlgorithm): string } /** - * Extracts the register index from the hash. - * - * The first `counterBits` bits of the hash determine the register. + * Extracts the register index from the 64-bit integer hash using bitwise shift. * - * @param string $hash binary hash - * @param int $counterBits number of bits used for indexing + * @param int $hash 64-bit integer hash + * @param int $counterBits number of bits used for indexing * * @return int register index */ - private function counter(string $hash, int $counterBits): int + private function counter(int $hash, int $counterBits): int { - $counter = 0; + $shift = 64 - $counterBits; + $mask = (1 << $counterBits) - 1; - for ($bit = 0; $bit < $counterBits; ++$bit) { - $byte = ord($hash[intdiv($bit, 8)]); - - $counter = ($counter << 1) - | (($byte >> (7 - ($bit % 8))) & 1); - } - - return $counter; + return ($hash >> $shift) & $mask; } /** - * Computes the position of the first set bit (rho function). + * Computes the position of the first set bit (rho function) using bitwise shifts. * * Counts leading zeros starting after the register bits. * - * @param string $hash binary hash - * @param int $counterBits number of bits reserved for register selection + * @param int $hash 64-bit integer hash + * @param int $counterBits number of bits reserved for register selection * * @return int position of first 1-bit (rho value) */ - private function rho(string $hash, int $counterBits): int + private function rho(int $hash, int $counterBits): int { - $bitLength = strlen($hash) * 8; - $rho = 1; + $maxBitsToCheck = 64 - $counterBits; - for ($bit = $counterBits; $bit < $bitLength; ++$bit) { - $byte = ord($hash[intdiv($bit, 8)]); + for ($i = 0; $i < $maxBitsToCheck; ++$i) { + $bitPos = 63 - $counterBits - $i; - if (($byte >> (7 - ($bit % 8))) & 1) { + if (($hash >> $bitPos) & 1) { return $rho; } diff --git a/tests/HyperLogLogTest.php b/tests/HyperLogLogTest.php index 15e8979..daac9da 100644 --- a/tests/HyperLogLogTest.php +++ b/tests/HyperLogLogTest.php @@ -16,11 +16,18 @@ class HyperLogLogTest extends TestCase { public function testConstructorSetsDefaultValues(): void { - $hll = new HyperLogLog(); + $hashAlgorithm = PHP_VERSION >= 8100 ? 'xxh3' : 'sha256'; + $hll = new HyperLogLog(hashAlgorithm: $hashAlgorithm); + + // Check if the default counter bits and hash algorithms are set $this->assertSame(5, $hll->getCounterBits()); - $this->assertSame('xxh3', $hll->getHashAlgorithm()); - $this->assertSame(32, $hll->getM()); // 1 << 5 = 32 + $this->assertSame($hashAlgorithm, $hll->getHashAlgorithm()); + + // m should be 2^5 = 32 + $this->assertSame(32, $hll->getM()); + + // The counters array should be initialized with 32 elements set to 0 $this->assertCount(32, $hll->getCounters()); } @@ -30,25 +37,30 @@ public function testConstructorSetsCustomValues(): void $this->assertSame(10, $hll->getCounterBits()); $this->assertSame('sha256', $hll->getHashAlgorithm()); - $this->assertSame(1024, $hll->getM()); // 1 << 10 = 1024 + + // m should be 2^10 = 1024 + $this->assertSame(1024, $hll->getM()); $this->assertCount(1024, $hll->getCounters()); } - public function testGettersAndSetters(): void + public function testConstructorFailsWithInvalidHashAlgorithm(): void { - $hll = new HyperLogLog(); - - $hll->setCounterBits(8); - $this->assertSame(8, $hll->getCounterBits()); + // In PHP 8+, the hash() function throws a ValueError if the provided algorithm does not exist. + $this->expectException(\InvalidArgumentException::class); + new HyperLogLog(10, 'invalid_algo_123'); + } - $hll->setHashAlgorithm('md5'); - $this->assertSame('md5', $hll->getHashAlgorithm()); + public function testGettersAndSetters(): void + { + $hashAlgorithm = PHP_VERSION >= 8100 ? 'xxh3' : 'sha256'; - $hll->setM(256); - $this->assertSame(256, $hll->getM()); + $hll = new HyperLogLog(hashAlgorithm: $hashAlgorithm); - $mockCounters = array_fill(0, 256, 1); + // Create a mock state to inject into the instance + $mockCounters = array_fill(0, 32, 1); $hll->setCounters($mockCounters); + + // Verify that the state was correctly updated $this->assertSame($mockCounters, $hll->getCounters()); } @@ -64,7 +76,7 @@ public function testAddAndCountUniqueElements(): void $estimate = $hll->count(); // With m=4096 (12 bits), the estimate should be highly accurate. - // We assert it falls within a reasonable probabilistic window (~5% error). + // We assert it falls within a reasonable probabilistic window (~5% error margin). $this->assertGreaterThan(4500, $estimate); $this->assertLessThan(5500, $estimate); } @@ -79,21 +91,86 @@ public function testAddDuplicateElementsMaintainsCardinality(): void $estimate = $hll->count(); - // Adding the exact same string 1000 times should result in a count of ~1 + // Adding the exact same string 1000 times should result in a total count of ~1 $this->assertEqualsWithDelta(1.0, $estimate, 1.0); } + public function testAddEmptyString(): void + { + $hll = new HyperLogLog(10, 'sha256'); + $hll->add(''); + + // An empty string is a valid element that should be processed and count as 1. + $this->assertEqualsWithDelta(1.0, $hll->count(), 1.0); + } + + public function testAddWithShortHashAlgorithmTriggersPaddingCorrectly(): void + { + // crc32 produces a 32-bit (4 bytes) hash. + // This must trigger the internal right-padding with null bytes so unpack('J') doesn't fail. + $hll = new HyperLogLog(10, 'crc32'); + + $hll->add('short_hash_test_1'); + $hll->add('short_hash_test_2'); + + $estimate = $hll->count(); + $this->assertGreaterThan(1.0, $estimate); + } + + public function testCountTriggersSmallRangeCorrectionInternalBranch(): void + { + $hll = new HyperLogLog(12, 'sha256'); // m = 4096 + + // By inserting only a few elements (e.g., 10), we ensure there are many empty registers ($V > 0) + // and that the raw estimate $E is <= (2.5 * 4096). + for ($i = 0; $i < 10; ++$i) { + $hll->add('small_range_item_'.$i); + } + + $estimate = $hll->count(); + + // Verify that the estimate remains consistent thanks to the linear counting correction. + $this->assertGreaterThan(5.0, $estimate); + $this->assertLessThan(15.0, $estimate); + } + + public function testCountTriggersLargeRangeCorrectionInternalBranch(): void + { + $hll = new HyperLogLog(10, 'sha256'); // m = 1024 + + // We use a simulated rho of 20 across all registers. + // The raw estimate $E will be approximately 773 million. + // This is well above the 143 million threshold (2^32 / 30) for large range correction, + // while remaining safely below the absolute 4.29 billion limit (2^32) to avoid logarithmic NAN errors. + $mockCounters = array_fill(0, 1024, 20); + $hll->setCounters($mockCounters); + + $estimate = $hll->count(); + $twoPow32 = 4_294_967_296; + + // Verify that the value is a valid float and not a mathematically undefined result (NAN). + $this->assertFalse(is_nan($estimate), 'The estimate must not be NAN.'); + + // Assert that the large range correction activated successfully and scaled the result appropriately. + $this->assertGreaterThan($twoPow32 / 30, $estimate); + } + public function testTheoreticalErrorRateCalculation(): void { - $hll = new HyperLogLog(); + $hashAlgorithm = PHP_VERSION >= 8100 ? 'xxh3' : 'sha256'; + $hll = new HyperLogLog(hashAlgorithm: $hashAlgorithm); + + // Theoretical error rate formula: 1.04 / sqrt(m) // 1.04 / sqrt(16) = 1.04 / 4 = 0.26 $this->assertEqualsWithDelta(0.26, $hll->theoreticalErrorRate(16), 0.001); } public function testTheoreticalErrorRateThrowsExceptionForInvalidM(): void { - $hll = new HyperLogLog(); + $hashAlgorithm = PHP_VERSION >= 8100 ? 'xxh3' : 'sha256'; + + $hll = new HyperLogLog(hashAlgorithm: $hashAlgorithm); $this->expectException(\InvalidArgumentException::class); $this->expectExceptionMessage('Invalid number of counters, $m must be greater than 0'); @@ -103,27 +180,35 @@ public function testTheoreticalErrorRateThrowsExceptionForInvalidM(): void public function testMeasureError(): void { - $hll = new HyperLogLog(); + $hashAlgorithm = PHP_VERSION >= 8100 ? 'xxh3' : 'sha256'; + + $hll = new HyperLogLog(hashAlgorithm: $hashAlgorithm); + // measureError simply returns the difference: estimate - real $this->assertSame(5, $hll->measureError(15, 10)); $this->assertSame(-2, $hll->measureError(8, 10)); } public function testAlphaCalculationValues(): void { - $hll = new HyperLogLog(); + $hashAlgorithm = PHP_VERSION >= 8100 ? 'xxh3' : 'sha256'; + $hll = new HyperLogLog(hashAlgorithm: $hashAlgorithm); + + // Test exact predefined constants for m=2 and m=16 $this->assertEqualsWithDelta(0.46852874309841, $hll->alpha(2), 0.0000001); $this->assertEqualsWithDelta(0.673, $hll->alpha(16), 0.001); - // Test default case calculation + // Test the default fallback calculation for m >= 128 (e.g., 512) $expectedLargeAlpha = 0.7213 / (1 + 1.079 / 512); $this->assertEqualsWithDelta($expectedLargeAlpha, $hll->alpha(512), 0.001); } public function testAlphaThrowsExceptionForInvalidM(): void { - $hll = new HyperLogLog(); + $hashAlgorithm = PHP_VERSION >= 8100 ? 'xxh3' : 'sha256'; + + $hll = new HyperLogLog(hashAlgorithm: $hashAlgorithm); $this->expectException(\InvalidArgumentException::class); $hll->alpha(0); @@ -131,21 +216,26 @@ public function testAlphaThrowsExceptionForInvalidM(): void public function testEstimateThrowsExceptionForInvalidZ(): void { - $hll = new HyperLogLog(); + $hashAlgorithm = PHP_VERSION >= 8100 ? 'xxh3' : 'sha256'; + + $hll = new HyperLogLog(hashAlgorithm: $hashAlgorithm); $this->expectException(\InvalidArgumentException::class); $this->expectExceptionMessage('Invalid harmonic mean, $Z must be positive'); + // $Z (Harmonic mean) cannot be zero or negative $hll->estimate(16, 0.0); } public function testEstimateUsingLinearCounting(): void { - $hll = new HyperLogLog(); + $hashAlgorithm = PHP_VERSION >= 8100 ? 'xxh3' : 'sha256'; + + $hll = new HyperLogLog(hashAlgorithm: $hashAlgorithm); $m = 64; - $v = 16; // 16 empty counters + $v = 16; // 16 empty counters out of 64 - // Linear Counting formula: m * log(m / V) + // Linear Counting formula for small cardinalities: m * log(m / V) $expected = 64 * log(64 / 16); // 64 * log(4) ≈ 88.72 $this->assertEqualsWithDelta($expected, $hll->estimateUsingLinearCounting($v, $m), 0.001); @@ -154,11 +244,15 @@ public function testEstimateUsingLinearCounting(): void public function testEstimateUsingLargeCardinalitiesApproach(): void { - $hll = new HyperLogLog(); + $hashAlgorithm = PHP_VERSION >= 8100 ? 'xxh3' : 'sha256'; + + $hll = new HyperLogLog(hashAlgorithm: $hashAlgorithm); + $rawEstimate = 3_000_000_000; $twoPow32 = 4_294_967_296; - // Large Cardinality formula: -2^32 * log(1 - E / 2^32) + // Large Cardinality formula to correct hash collision bias near 2^32: + // -2^32 * log(1 - E / 2^32) $expected = -$twoPow32 * log(1 - $rawEstimate / $twoPow32); $this->assertEqualsWithDelta(