diff --git a/README.md b/README.md index 578b755..c6a0506 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ bitmap.addRange(min: 0, max: 500) let cpy = bitmap.copy() //Example: Operators -let and = bitmap && cpy +let and = bitmap & cpy //Example: Iterate for i in bitmap { @@ -96,6 +96,29 @@ for i in bitmap { ``` +### 64-bit bitmaps + +`RoaringBitmap` stores `UInt32` values. To store `UInt64` values, use +`Roaring64Bitmap`, which offers the same API: + +```swift +import SwiftRoaring + +let bitmap: Roaring64Bitmap = [1, 1 << 32, UInt64.max] + +bitmap.add(1_000...2_000) +print(bitmap.count) // 1004 +print(bitmap.contains(1 << 32)) // true + +//Set operations, ranges, rank/select, iterators and +//serialization all work as they do in 32 bits. +let other = Roaring64Bitmap(range: 0..<1_000_000, step: 7) +let common = bitmap & other + +//Convert a 32-bit bitmap to a 64-bit one +let widened = Roaring64Bitmap(RoaringBitmap([1, 2, 3])) +``` + ### Development You can build using Swift Package Manager as follows: diff --git a/Sources/CRoaring/include/roaring.h b/Sources/CRoaring/include/roaring.h index 62ce1a8..b5aa180 100644 --- a/Sources/CRoaring/include/roaring.h +++ b/Sources/CRoaring/include/roaring.h @@ -1,5 +1,5 @@ // !!! DO NOT EDIT - THIS IS AN AUTO-GENERATED FILE !!! -// Created by amalgamation.sh on 2025-12-05T04:42:38Z +// Created by amalgamation.sh on 2026-08-22T02:17:56Z /* * The CRoaring project is under a dual license (Apache/MIT). @@ -59,10 +59,10 @@ // /include/roaring/roaring_version.h automatically generated by release.py, do not change by hand #ifndef ROARING_INCLUDE_ROARING_VERSION #define ROARING_INCLUDE_ROARING_VERSION -#define ROARING_VERSION "4.5.0" +#define ROARING_VERSION "5.1.0" enum { - ROARING_VERSION_MAJOR = 4, - ROARING_VERSION_MINOR = 5, + ROARING_VERSION_MAJOR = 5, + ROARING_VERSION_MINOR = 1, ROARING_VERSION_REVISION = 0 }; #endif // ROARING_INCLUDE_ROARING_VERSION @@ -71,12 +71,21 @@ enum { /* * portability.h * + * This header centralizes compiler-, platform-, and architecture-specific + * portability definitions used throughout CRoaring. It provides feature + * detection, calling-convention and attribute macros, intrinsic and inline + * assembly enablement, endianness helpers, alignment annotations, atomic + * reference-count support, and other low-level compatibility glue. + * + * The goal is to keep these conditional definitions in one place so the rest + * of the codebase can rely on a more uniform interface across GCC, Clang, + * MSVC, x86/x64, ARM/NEON, and other supported environments. */ /** * All macros should be prefixed with either CROARING or ROARING. * The library uses both ROARING_... - * as well as CROAIRING_ as prefixes. The ROARING_ prefix is for + * as well as CROARING_ as prefixes. The ROARING_ prefix is for * macros that are provided by the build system or that are closely * related to the format. The header macros may also use ROARING_. * The CROARING_ prefix is for internal macros that a user is unlikely @@ -125,6 +134,12 @@ enum { #ifdef __GLIBC__ #include // this should never be needed but there are some reports that it is needed. #endif +// alignas/alignof are keywords in C++ and in C23+, where is +// deprecated. Only include it for C11..C17. +#if !defined(__cplusplus) && \ + (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 202311L) +#include +#endif #ifdef __cplusplus extern "C" { // portability definitions are in global scope, not a namespace @@ -140,7 +155,14 @@ extern "C" { // portability definitions are in global scope, not a namespace #endif // __restrict__ #endif // CROARING_REGULAR_VISUAL_STUDIO -#if defined(__x86_64__) || defined(_M_X64) +#if defined(__riscv) || defined(_M_RISCV32) || defined(_M_RISCV64) +#define CROARING_IS_RISCV 1 + +#if (defined(__riscv_xlen) && (__riscv_xlen == 64)) || defined(_M_RISCV64) +#define CROARING_IS_RISCV64 1 +#endif + +#elif defined(__x86_64__) || defined(_M_X64) // we have an x64 processor #define CROARING_IS_X64 1 @@ -225,7 +247,7 @@ extern "C" { // portability definitions are in global scope, not a namespace #define CROARING_IS_E2K 1 #endif -#if !CROARING_REGULAR_VISUAL_STUDIO && !defined(CROARING_IS_E2K) +#if !CROARING_REGULAR_VISUAL_STUDIO && !defined(CROARING_IS_E2K) && !__FILC__ /* Non-Microsoft C/C++-compatible compiler, assumes that it supports inline * assembly */ #define CROARING_INLINE_ASM 1 @@ -523,6 +545,55 @@ static inline int roaring_hamming(uint64_t x) { #define croaring_be64toh(x) croaring_htobe64(x) // End of host <-> big endian conversion. +// Host <-> little-endian conversion helpers. +// +// The CRoaring "portable" serialization format (and the regular +// roaring_bitmap_serialize / Roaring64Map::write formats which build on it) +// is defined to be little-endian on the wire. Code that reads or writes +// multi-byte integers to such buffers must convert between host and +// little-endian byte order. On little-endian hosts these are no-ops; on +// big-endian hosts they swap bytes. +// +// The "frozen" format is intentionally non-portable and uses native byte +// order; it must not use these helpers. +#if CROARING_IS_BIG_ENDIAN + +static inline uint16_t croaring_bswap16(uint16_t x) { + return (uint16_t)((x << 8) | (x >> 8)); +} + +static inline uint32_t croaring_bswap32(uint32_t x) { + return ((x & 0x000000FFU) << 24) | ((x & 0x0000FF00U) << 8) | + ((x & 0x00FF0000U) >> 8) | ((x & 0xFF000000U) >> 24); +} + +static inline uint64_t croaring_bswap64(uint64_t x) { + return ((x & 0x00000000000000FFULL) << 56) | + ((x & 0x000000000000FF00ULL) << 40) | + ((x & 0x0000000000FF0000ULL) << 24) | + ((x & 0x00000000FF000000ULL) << 8) | + ((x & 0x000000FF00000000ULL) >> 8) | + ((x & 0x0000FF0000000000ULL) >> 24) | + ((x & 0x00FF000000000000ULL) >> 40) | + ((x & 0xFF00000000000000ULL) >> 56); +} + +#define croaring_htole16(x) croaring_bswap16(x) +#define croaring_htole32(x) croaring_bswap32(x) +#define croaring_htole64(x) croaring_bswap64(x) + +#else // CROARING_IS_BIG_ENDIAN + +#define croaring_htole16(x) (x) +#define croaring_htole32(x) (x) +#define croaring_htole64(x) (x) + +#endif // CROARING_IS_BIG_ENDIAN + +#define croaring_letoh16(x) croaring_htole16(x) +#define croaring_letoh32(x) croaring_htole32(x) +#define croaring_letoh64(x) croaring_htole64(x) + // Defines for the possible CROARING atomic implementations #define CROARING_ATOMIC_IMPL_NONE 1 #define CROARING_ATOMIC_IMPL_CPP 2 @@ -538,13 +609,13 @@ static inline int roaring_hamming(uint64_t x) { #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP #endif //__has_include() #else - // We lack __has_include to check: +// We lack __has_include to check: #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP #endif //__has_include #elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C #elif CROARING_REGULAR_VISUAL_STUDIO - // https://www.technetworkhub.com/c11-atomics-in-visual-studio-2022-version-17/ +// https://www.technetworkhub.com/c11-atomics-in-visual-studio-2022-version-17/ #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C_WINDOWS #endif #endif // !defined(CROARING_ATOMIC_IMPL) @@ -634,7 +705,7 @@ static inline void croaring_refcount_inc(croaring_refcount_t *val) { static inline bool croaring_refcount_dec(croaring_refcount_t *val) { assert(*val > 0); *val -= 1; - return val == 0; + return *val == 0; } static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { @@ -678,9 +749,22 @@ static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { #endif /* INCLUDE_PORTABILITY_H_ */ /* end file include/roaring/portability.h */ /* begin file include/roaring/isadetection.h */ +/* + * isadetection.h + * + * This header declares the small interface used to detect instruction-set + * capabilities relevant to CRoaring's optimized kernels on x64 platforms. It + * also defines compile-time feature macros that indicate whether the compiler + * toolchain is capable of building AVX-512 code paths. + * + * The resulting flags are used to decide whether accelerated implementations, + * such as AVX2 or AVX-512 variants, can be selected safely at runtime. + */ #ifndef ROARING_ISADETECTION_H #define ROARING_ISADETECTION_H -#if defined(__x86_64__) || defined(_M_AMD64) // x64 + + +#if CROARING_IS_X64 // x64 #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #ifdef __has_include @@ -718,12 +802,18 @@ int croaring_hardware_support(void); } } // extern "C" { namespace roaring { namespace internal { #endif -#endif // x64 +#endif // CROARING_IS_X64 #endif // ROARING_ISADETECTION_H /* end file include/roaring/isadetection.h */ /* begin file include/roaring/roaring_types.h */ /* - Typedefs used by various components + Shared type definitions used across the CRoaring public API and internal + components. + + This file centralizes common typedefs and small structs that are referenced + by multiple headers, including iterator callback signatures, roaring array + metadata, statistics structures, and compatibility types used to bridge the + C and C++ builds. */ #ifndef ROARING_TYPES_H @@ -762,6 +852,8 @@ struct container_s {}; #define ROARING_FLAG_COW UINT8_C(0x1) #define ROARING_FLAG_FROZEN UINT8_C(0x2) +// 64-bit only: ART node arrays alias a frozen buffer (do not art_free). +#define ROARING_FLAG_FROZEN_ART UINT8_C(0x4) /** * Roaring arrays are array-based key-value pairs having containers as values @@ -876,17 +968,31 @@ typedef struct roaring_container_iterator_s { #endif /* ROARING_TYPES_H */ /* end file include/roaring/roaring_types.h */ /* begin file include/roaring/bitset/bitset.h */ +/* + * bitset.h + * + * This bitset is a general-purpose dynamic bitmap storing bits in a contiguous + * array of 64-bit words. The array field points to the word buffer, arraysize + * records how many words are currently in use, and capacity records how many + * words are allocated. + * + * Unlike the fixed 16-bit-domain container bitset, this structure can grow to + * cover an arbitrary number of bit positions. It is useful when callers need a + * resizable bitmap with efficient bitwise operations, scans, and shifts over a + * larger or runtime-defined domain. + */ #ifndef CROARING_CBITSET_BITSET_H #define CROARING_CBITSET_BITSET_H // For compatibility with MSVC with the use of `restrict` -#if (__STDC_VERSION__ >= 199901L) || \ +#ifdef __cplusplus +#define CROARING_CBITSET_RESTRICT +#elif (__STDC_VERSION__ >= 199901L) || \ (defined(__GNUC__) && defined(__STDC_VERSION__)) #define CROARING_CBITSET_RESTRICT restrict #else #define CROARING_CBITSET_RESTRICT -#endif // (__STDC_VERSION__ >= 199901L) || (defined(__GNUC__) && - // defined(__STDC_VERSION__ )) +#endif #include #include @@ -1275,6 +1381,18 @@ PPDerived movable_CAST_HELPER(Base **ptr_to_ptr) { #endif /* INCLUDE_CONTAINERS_CONTAINER_DEFS_H_ */ /* end file include/roaring/containers/container_defs.h */ /* begin file include/roaring/array_util.h */ +/* + * array_util.h + * + * This header provides low-level utility routines for sorted arrays of + * 16-bit integers, which are used heavily by CRoaring's array-based + * containers and set-operation kernels. It includes search helpers, counting + * helpers, and array intersection/difference primitives. + * + * Some of the routines also have SIMD-accelerated implementations on supported + * platforms, allowing efficient operations on sorted integer arrays that form + * the basis of sparse container processing. + */ #ifndef CROARING_ARRAY_UTIL_H #define CROARING_ARRAY_UTIL_H @@ -1299,29 +1417,95 @@ namespace internal { #endif /* - * Good old binary search. + * Sorted-array search. * Assumes that array is sorted, has logarithmic complexity. * if the result is x, then: * if ( x>0 ) you have array[x] = ikey * if ( x<0 ) then inserting ikey at position -x-1 in array (insuring that * array[-x-1]=ikey) keys the array sorted. + * + * Adapted from array_container_contains: a SIMD-quad block-narrowing + * search at gap=16 (Daniel Lemire, + * https://lemire.me/blog/2026/04/27/you-can-beat-the-binary-search/) + * followed by a scalar in-block scan that recovers the exact insertion + * point required by the binarySearch contract. */ inline int32_t binarySearch(const uint16_t *array, int32_t lenarray, uint16_t ikey) { - int32_t low = 0; - int32_t high = lenarray - 1; - while (low <= high) { - int32_t middleIndex = (low + high) >> 1; - uint16_t middleValue = array[middleIndex]; - if (middleValue < ikey) { - low = middleIndex + 1; - } else if (middleValue > ikey) { - high = middleIndex - 1; - } else { - return middleIndex; + const int32_t gap = 16; + if (lenarray < gap) { + for (int32_t j = 0; j < lenarray; j++) { + if (array[j] >= ikey) { + return (array[j] == ikey) ? j : -(j + 1); + } + } + return -(lenarray + 1); + } + const int32_t num_blocks = lenarray / gap; + int32_t base = 0; + int32_t n = num_blocks; + while (n > 3) { + int32_t quarter = n >> 2; + + int32_t k1 = array[(base + quarter + 1) * gap - 1]; + int32_t k2 = array[(base + 2 * quarter + 1) * gap - 1]; + int32_t k3 = array[(base + 3 * quarter + 1) * gap - 1]; + + int32_t c1 = (k1 < ikey); + int32_t c2 = (k2 < ikey); + int32_t c3 = (k3 < ikey); + + base += (c1 + c2 + c3) * quarter; + n -= 3 * quarter; + } + while (n > 1) { + int32_t half = n >> 1; + base = (array[(base + half + 1) * gap - 1] < ikey) ? base + half : base; + n -= half; + } + int32_t lo = (array[(base + 1) * gap - 1] < ikey) ? base + 1 : base; + + if (lo < num_blocks) { + const int32_t start = lo * gap; +#if defined(CROARING_IS_X64) + // SSE2: subs_epu16 yields zero where lane >= ikey. movemask of an + // epi16 compare gives 2 bits per lane; ctz>>1 = lane index. Scan + // the first 8 lanes first and exit early when they contain the + // answer; otherwise the block-narrowing invariant guarantees the + // second-half mask is non-zero. + __m128i needle = _mm_set1_epi16((short)ikey); + __m128i zero = _mm_setzero_si128(); + __m128i v0 = _mm_loadu_si128((const __m128i *)(array + start)); + __m128i ge0 = _mm_cmpeq_epi16(_mm_subs_epu16(needle, v0), zero); + unsigned m0 = (unsigned)_mm_movemask_epi8(ge0); + if (m0 != 0) { + int32_t j = start + (int32_t)(roaring_trailing_zeroes(m0) >> 1); + return (array[j] == ikey) ? j : -(j + 1); + } + __m128i v1 = _mm_loadu_si128((const __m128i *)(array + start + 8)); + __m128i ge1 = _mm_cmpeq_epi16(_mm_subs_epu16(needle, v1), zero); + unsigned m1 = (unsigned)_mm_movemask_epi8(ge1); + int32_t j = start + 8 + (int32_t)(roaring_trailing_zeroes(m1) >> 1); + return (array[j] == ikey) ? j : -(j + 1); +#else + const int32_t end = start + gap; + for (int32_t j = start; j < end; j++) { + if (array[j] >= ikey) { + return (array[j] == ikey) ? j : -(j + 1); + } } + // Unreachable: the narrowing guarantees the last element of the + // selected block is >= ikey. + return -(end + 1); +#endif } - return -(low + 1); + + for (int32_t j = num_blocks * gap; j < lenarray; j++) { + if (array[j] >= ikey) { + return (array[j] == ikey) ? j : -(j + 1); + } + } + return -(lenarray + 1); } /** @@ -1409,12 +1593,11 @@ static inline int32_t count_greater(const uint16_t *array, int32_t lenarray, * C should have capacity greater than the minimum of s_1 and s_b + 8 * where 8 is sizeof(__m128i)/sizeof(uint16_t). */ -int32_t intersect_vector16(const uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b, - uint16_t *C); +int32_t intersect_vector16(const uint16_t *A, size_t s_a, const uint16_t *B, + size_t s_b, uint16_t *C); -int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b); +int32_t intersect_vector16_inplace(uint16_t *A, size_t s_a, const uint16_t *B, + size_t s_b); /** * Take an array container and write it out to a 32-bit array, using base @@ -1429,10 +1612,8 @@ int avx512_array_container_to_uint32_array(void *vout, const uint16_t *array, /** * Compute the cardinality of the intersection using SSE4 instructions */ -int32_t intersect_vector16_cardinality(const uint16_t *__restrict__ A, - size_t s_a, - const uint16_t *__restrict__ B, - size_t s_b); +int32_t intersect_vector16_cardinality(const uint16_t *A, size_t s_a, + const uint16_t *B, size_t s_b); /* Computes the intersection between one small and one large set of uint16_t. * Stores the result into buffer and return the number of elements. */ @@ -1507,22 +1688,33 @@ size_t union_uint32(const uint32_t *set_1, size_t size_1, const uint32_t *set_2, /** * A fast SSE-based union function. */ -uint32_t union_vector16(const uint16_t *__restrict__ set_1, uint32_t size_1, - const uint16_t *__restrict__ set_2, uint32_t size_2, - uint16_t *__restrict__ buffer); +uint32_t union_vector16(const uint16_t *set_1, uint32_t size_1, + const uint16_t *set_2, uint32_t size_2, + uint16_t *buffer); + +#if CROARING_COMPILER_SUPPORTS_AVX512 +/** + * AVX-512 union of two sorted uint16 arrays, using a Batcher bitonic merge + * network over 32 lanes. Same contract as union_vector16: `buffer` must have + * room for size_1 + size_2 values. + */ +uint32_t avx512_union_uint16(const uint16_t *set_1, uint32_t size_1, + const uint16_t *set_2, uint32_t size_2, + uint16_t *buffer); +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + /** * A fast SSE-based XOR function. */ -uint32_t xor_vector16(const uint16_t *__restrict__ array1, uint32_t length1, - const uint16_t *__restrict__ array2, uint32_t length2, - uint16_t *__restrict__ output); +uint32_t xor_vector16(const uint16_t *array1, uint32_t length1, + const uint16_t *array2, uint32_t length2, + uint16_t *output); /** * A fast SSE-based difference function. */ -int32_t difference_vector16(const uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b, - uint16_t *C); +int32_t difference_vector16(const uint16_t *A, size_t s_a, const uint16_t *B, + size_t s_b, uint16_t *C); /** * Generic union function, returns just the cardinality. @@ -1550,6 +1742,19 @@ bool memequals(const void *s1, const void *s2, size_t n); #endif /* end file include/roaring/array_util.h */ /* begin file include/roaring/bitset_util.h */ +/* + * bitset_util.h + * + * This header collects low-level utility functions for operating on raw + * bitsets represented as arrays of 64-bit words. It includes helpers for + * setting, clearing, flipping, counting, extracting, and combining bit ranges, + * along with architecture-specific SIMD implementations for performance- + * critical routines. + * + * In particular, this file contains accelerated extraction and population- + * count code paths for x64 targets using AVX2 and, when supported by the + * compiler and hardware, AVX-512. + */ #ifndef CROARING_BITSET_UTIL_H #define CROARING_BITSET_UTIL_H @@ -2271,6 +2476,14 @@ CROARING_UNTARGET_AVX512 /* * array.h * + * Array containers store a sparse set of 16-bit integers as a sorted dynamic + * array. The cardinality field tracks how many values are present, capacity + * tracks the allocated length, and array points to the sorted values. + * + * This representation is used for low-cardinality containers because it is + * compact and supports fast iteration and binary-search-based membership + * tests. When the number of stored values grows beyond DEFAULT_MAX_SIZE, + * CRoaring will typically switch to a denser container representation. */ #ifndef INCLUDE_CONTAINERS_ARRAY_H_ @@ -2599,30 +2812,72 @@ static inline bool array_container_remove(array_container_t *arr, /* Check whether x is present. */ inline bool array_container_contains(const array_container_t *arr, uint16_t pos) { - // return binarySearch(arr->array, arr->cardinality, pos) >= 0; - // binary search with fallback to linear search for short ranges - int32_t low = 0; - const uint16_t *carr = (const uint16_t *)arr->array; - int32_t high = arr->cardinality - 1; - // while (high - low >= 0) { - while (high >= low + 16) { - int32_t middleIndex = (low + high) >> 1; - uint16_t middleValue = carr[middleIndex]; - if (middleValue < pos) { - low = middleIndex + 1; - } else if (middleValue > pos) { - high = middleIndex - 1; - } else { - return true; + /** + * SIMD Quad algorithm + * Daniel Lemire, "You can beat the binary search," in Daniel Lemire's blog, + * April 27, 2026, + * https://lemire.me/blog/2026/04/27/you-can-beat-the-binary-search/. + */ + const int32_t gap = 16; + const uint16_t *carr = arr->array; + int32_t cardinality = arr->cardinality; + if (cardinality < gap) { + for (int32_t j = 0; j < cardinality; j++) { + if (carr[j] >= pos) return carr[j] == pos; } + return false; } + int32_t num_blocks = cardinality / gap; + int32_t base = 0; + int32_t n = num_blocks; + while (n > 3) { + int32_t quarter = n >> 2; - for (int i = low; i <= high; i++) { - uint16_t v = carr[i]; - if (v == pos) { - return true; + int32_t k1 = carr[(base + quarter + 1) * gap - 1]; + int32_t k2 = carr[(base + 2 * quarter + 1) * gap - 1]; + int32_t k3 = carr[(base + 3 * quarter + 1) * gap - 1]; + + int32_t c1 = (k1 < pos); + int32_t c2 = (k2 < pos); + int32_t c3 = (k3 < pos); + + base += (c1 + c2 + c3) * quarter; + n -= 3 * quarter; + } + while (n > 1) { + int32_t half = n >> 1; + base = (carr[(base + half + 1) * gap - 1] < pos) ? base + half : base; + n -= half; + } + int32_t lo = (carr[(base + 1) * gap - 1] < pos) ? base + 1 : base; + + if (lo < num_blocks) { + const uint16_t *blk = carr + lo * gap; +#ifdef CROARING_USENEON + uint16x8_t needle = vdupq_n_u16(pos); + uint16x8_t v0 = vld1q_u16(blk); + uint16x8_t v1 = vld1q_u16(blk + 8); + uint16x8_t hit = + vorrq_u16(vceqq_u16(v0, needle), vceqq_u16(v1, needle)); + return vmaxvq_u16(hit) != 0; +#elif defined(CROARING_IS_X64) + __m128i needle = _mm_set1_epi16((short)pos); + __m128i v0 = _mm_loadu_si128((const __m128i *)blk); + __m128i v1 = _mm_loadu_si128((const __m128i *)(blk + 8)); + __m128i hit = _mm_or_si128(_mm_cmpeq_epi16(v0, needle), + _mm_cmpeq_epi16(v1, needle)); + return _mm_movemask_epi8(hit) != 0; +#else + for (int32_t j = 0; j < gap; j++) { + if (blk[j] >= pos) return blk[j] == pos; } - if (v > pos) return false; + return false; +#endif + } + + for (int32_t j = num_blocks * gap; j < cardinality; j++) { + uint16_t v = carr[j]; + if (v >= pos) return (v == pos); } return false; } @@ -2790,6 +3045,15 @@ static inline void array_container_remove_range(array_container_t *array, /* * bitset.h * + * Bitset containers store a set of 16-bit integers as a fixed-size bitmap. + * The words pointer references an array of 64-bit words covering the full + * 16-bit domain, with one bit per possible value. The cardinality field tracks + * the number of set bits; when it is BITSET_UNKNOWN_CARDINALITY, the count must + * be recomputed from the bitmap contents. + * + * This representation is used for denser containers because membership tests, + * set operations, and sequential scans can be implemented efficiently with + * word-level bitwise operations. */ #ifndef INCLUDE_CONTAINERS_BITSET_H_ @@ -2831,6 +3095,10 @@ typedef struct bitset_container_s bitset_container_t; /* Create a new bitset. Return NULL in case of failure. */ bitset_container_t *bitset_container_create(void); +/* Create a bitset without zeroing the words. Caller must overwrite `words` + * before the container is used. Return NULL in case of failure. */ +bitset_container_t *bitset_container_create_uninitialized(void); + /* Free memory. */ void bitset_container_free(bitset_container_t *bitset); @@ -3301,6 +3569,14 @@ int bitset_container_index_equalorlarger(const bitset_container_t *container, /* * run.h * + * Run containers store a set of 16-bit integers as a sorted array of + * non-overlapping runs. Each run is represented by a starting value and a + * length, encoding one contiguous interval of present integers. + * + * This representation is effective when the data contains long consecutive + * ranges because it compresses many adjacent values into a small number of + * run records while still supporting search and set operations over the + * interval list. */ #ifndef INCLUDE_CONTAINERS_RUN_H_ @@ -4017,6 +4293,15 @@ static inline void run_container_remove_range(run_container_t *run, /* * convert.h * + * This header declares conversion helpers between the different Roaring + * container representations: array, bitset, and run containers. These + * routines are used when an operation produces data better represented in a + * different form, or when the library wants to switch to the most space- + * efficient container type. + * + * In addition to direct conversions, the file also provides helpers that + * choose between candidate result representations based on cardinality and + * storage efficiency. */ #ifndef INCLUDE_CONTAINERS_CONVERT_H_ @@ -4130,6 +4415,12 @@ bool run_container_equals_bitset(const run_container_t* container1, /* * mixed_subset.h * + * This header declares subset-checking routines between different Roaring + * container types. These helpers are used when two containers do not share the + * same representation and a direct type-specific subset predicate is needed. + * + * Each function answers whether all values from one container are contained in + * another, across combinations of array, bitset, and run containers. */ #ifndef CONTAINERS_MIXED_SUBSET_H_ @@ -4183,6 +4474,16 @@ bool bitset_container_is_subset_run(const bitset_container_t* container1, /* begin file include/roaring/containers/mixed_andnot.h */ /* * mixed_andnot.h + * + * This header declares mixed-container difference operations of the form + * `A \ B` (also called andnot) between Roaring container types such as array, + * bitset, and run containers. These helpers are used when the operands have + * different internal representations and the result may need to change + * representation depending on density. + * + * The file includes both allocating and inplace-oriented variants so callers + * can either materialize a fresh result or reuse storage when that is + * efficient and semantically allowed. */ #ifndef INCLUDE_CONTAINERS_MIXED_ANDNOT_H_ #define INCLUDE_CONTAINERS_MIXED_ANDNOT_H_ @@ -4364,6 +4665,15 @@ bool bitset_bitset_container_iandnot(bitset_container_t *src_1, /* * mixed_intersection.h * + * This header declares intersection operations between different Roaring + * container types, such as array, bitset, and run containers. These mixed + * routines are used when the input containers have different internal + * representations and the implementation must choose the appropriate result + * form based on the data. + * + * In addition to materializing intersections, the file also provides helpers + * for intersection cardinality, intersection predicates, and selected inplace + * variants. */ #ifndef INCLUDE_CONTAINERS_MIXED_INTERSECTION_H_ @@ -4464,6 +4774,15 @@ bool bitset_bitset_container_intersection_inplace( /* * mixed_negation.h * + * This header declares negation (complement) operations for Roaring + * containers, both over the full 16-bit container domain and over specified + * subranges. Depending on the input representation and the density of the + * complement, the result may need to switch between array, bitset, and run + * containers. + * + * The file includes both allocating and inplace-oriented variants so callers + * can choose between simple result construction and reuse of an existing + * container when that is practical. */ #ifndef INCLUDE_CONTAINERS_MIXED_NEGATION_H_ @@ -4601,8 +4920,16 @@ int run_container_negation_range_inplace(run_container_t *src, /* end file include/roaring/containers/mixed_negation.h */ /* begin file include/roaring/containers/mixed_union.h */ /* - * mixed_intersection.h + * mixed_union.h * + * This header declares union operations between different Roaring container + * types, such as array, bitset, and run containers. These mixed-operation + * helpers are used when the two input containers do not share the same + * representation and the result may need to stay in, or be converted to, a + * representation chosen according to the data. + * + * The file includes regular, lazy, and inplace variants so callers can select + * between fully maintained results and faster deferred-maintenance paths. */ #ifndef INCLUDE_CONTAINERS_MIXED_UNION_H_ @@ -4719,6 +5046,14 @@ void run_bitset_container_lazy_union(const run_container_t *src_1, /* * mixed_xor.h * + * This header declares XOR operations between different Roaring container + * types, such as array, bitset, and run containers. These "mixed" routines + * handle cases where the two inputs do not share the same representation and + * where the most appropriate output representation may depend on the data. + * + * It includes regular, lazy, and inplace variants so higher-level bitmap code + * can choose between fully normalized results and faster deferred-maintenance + * paths. */ #ifndef INCLUDE_CONTAINERS_MIXED_XOR_H_ @@ -4891,6 +5226,18 @@ int run_run_container_ixor(run_container_t *src_1, const run_container_t *src_2, #endif /* end file include/roaring/containers/mixed_xor.h */ /* begin file include/roaring/containers/containers.h */ +/* + * containers.h + * + * This header is the central internal interface for Roaring container + * operations. It ties together the concrete container types (array, bitset, + * run, and shared containers), their type codes, common helper functions, and + * the mixed-operation headers used to combine different representations. + * + * In practice, it acts as the dispatch layer that lets higher-level bitmap + * code manipulate containers through a uniform interface while still selecting + * type-specific implementations when needed. + */ #ifndef CONTAINERS_CONTAINERS_H #define CONTAINERS_CONTAINERS_H @@ -5429,13 +5776,18 @@ static inline container_t *container_remove( } /** - * Check whether a value is in a container, requires a typecode + * Check whether a value is in a container, requires a typecode */ -static inline bool container_contains( +inline bool container_contains( const container_t *c, uint16_t val, uint8_t typecode // !!! should be second argument? ) { - c = container_unwrap_shared(c, &typecode); + if (typecode == SHARED_CONTAINER_TYPE) { + typecode = const_CAST_shared(c)->typecode; + assert(typecode != SHARED_CONTAINER_TYPE); + c = const_CAST_shared(c)->container; + } + switch (typecode) { case BITSET_CONTAINER_TYPE: return bitset_container_get(const_CAST_bitset(c), val); @@ -7315,6 +7667,7 @@ roaring_container_iterator_t container_init_iterator_last(const container_t *c, * Moves the iterator to the next entry. Returns true and sets `value` if a * value is present. */ +CROARING_ALLOW_UNALIGNED inline bool container_iterator_next(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint16_t *value) { @@ -7383,6 +7736,7 @@ inline bool container_iterator_next(const container_t *c, uint8_t typecode, * Moves the iterator to the previous entry. Returns true and sets `value` if a * value is present. */ +CROARING_ALLOW_UNALIGNED inline bool container_iterator_prev(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint16_t *value) { @@ -7474,6 +7828,32 @@ bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, uint32_t count, uint32_t *consumed, uint16_t *value_out); +/** + * Reads up to `count` entries backward from the container, writing them into + * `buf` as `high16 | entry` in descending order. Returns true and sets + * `value_out` if a value is present before the entries read. Sets `consumed` + * to the number of values read. `count` should be greater than zero. + * + * `value_out` must be initialized to the current value yielded by the iterator. + */ +bool container_iterator_read_backward_into_uint32( + const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, + uint32_t high16, uint32_t *buf, uint32_t count, uint32_t *consumed, + uint16_t *value_out); + +/** + * Reads up to `count` entries backward from the container, writing them into + * `buf` as `high48 | entry` in descending order. Returns true and sets + * `value_out` if a value is present before the entries read. Sets `consumed` + * to the number of values read. `count` should be greater than zero. + * + * `value_out` must be initialized to the current value yielded by the iterator. + */ +bool container_iterator_read_backward_into_uint64( + const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, + uint64_t high48, uint64_t *buf, uint32_t count, uint32_t *consumed, + uint16_t *value_out); + /** * Skips the next `skip_count` entries in the container iterator. Returns true * and sets `value_out` if a value is present after skipping. Returns false if @@ -7508,6 +7888,35 @@ bool container_iterator_skip_backward(const container_t *c, uint8_t typecode, uint32_t *consumed_count, uint16_t *value_out); +/** + * Finds the end of the consecutive run starting at the current iterator + * position within a container. Returns the low16 of the last consecutive + * value. If there are more values in the container after the run, + * *has_more is set to true, the iterator is positioned at the next value, + * and *value is updated to that value. Otherwise *has_more is set to false. + * + * *value must be the low 16 bits of the current value at the iterator's + * position on entry. + */ +uint16_t container_iterator_find_run_end(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value, bool *has_more); + +/** + * Finds the start of the consecutive run ending at the current iterator + * position within a container. Returns the low16 of the first consecutive + * value. If there are more values in the container before the run, + * *has_more is set to true, the iterator is positioned at the previous value, + * and *value is updated to that value. Otherwise *has_more is set to false. + * + * *value must be the low 16 bits of the current value at the iterator's + * position on entry. + */ +uint16_t container_iterator_find_run_start(const container_t *c, + uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value, bool *has_more); + #ifdef __cplusplus } } @@ -7517,6 +7926,20 @@ bool container_iterator_skip_backward(const container_t *c, uint8_t typecode, #endif /* end file include/roaring/containers/containers.h */ /* begin file include/roaring/roaring_array.h */ +/* + * roaring_array.h + * + * This file declares the roaring_array helper structure and the operations + * used to manage it. A roaring array is the top-level index used by a 32-bit + * Roaring bitmap: it stores sorted 16-bit high keys alongside the container + * pointers and type codes associated with each key. + * + * In effect, it is the directory that maps each populated 16-bit chunk of the + * 32-bit value space to the container holding that chunk's low 16-bit values. + * The functions in this header handle allocation, lookup, insertion, + * replacement, copying, serialization support, and structural updates on that + * directory. + */ #ifndef INCLUDE_ROARING_ARRAY_H #define INCLUDE_ROARING_ARRAY_H @@ -7821,6 +8244,17 @@ void ra_shift_tail(roaring_array_t *ra, int32_t count, int32_t distance); /* begin file include/roaring/roaring.h */ /* * An implementation of Roaring Bitmaps in C. + * + * This is the main public header for the 32-bit CRoaring API. A Roaring bitmap + * represents a set of unsigned 32-bit integers by partitioning the value space + * into 16-bit chunks and storing each chunk in a container chosen to match the + * local data density. Sparse chunks are typically kept as sorted arrays, + * denser chunks as bitsets, and long consecutive runs as run containers. + * + * This hybrid representation aims to keep bitmaps compact while still + * supporting fast membership tests, iteration, rank/select queries, + * serialization, and set operations such as union, intersection, difference, + * and symmetric difference. */ #ifndef ROARING_H @@ -8439,10 +8873,6 @@ size_t roaring_bitmap_shrink_to_fit(roaring_bitmap_t *r); * * Returns how many bytes written, should be `roaring_bitmap_size_in_bytes(r)`. * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * When serializing data to a file, we recommend that you also use * checksums so that, at deserialization, you can be confident * that you are recovering the correct data. @@ -8455,27 +8885,34 @@ size_t roaring_bitmap_serialize(const roaring_bitmap_t *r, char *buf); * (See `roaring_bitmap_portable_deserialize()` if you want a format that's * compatible with Java and Go implementations). * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * The returned pointer may be NULL in case of errors. */ roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf); /** + * Load a bitmap from a serialized buffer safely (reading up to maxbytes). + * * Use with `roaring_bitmap_serialize()`. * * (See `roaring_bitmap_portable_deserialize_safe()` if you want a format that's * compatible with Java and Go implementations). * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * The difference with `roaring_bitmap_deserialize()` is that this function - * checks that the input buffer is a valid bitmap. If the buffer is too small, - * NULL is returned. + * is guaranteed to not read beyond the provided buffer. If the buffer is too + * small, NULL is returned. + * + * The function itself is safe in the sense that it will not cause buffer + * overflows: it will not read beyond the scope of the provided buffer + * (buf,maxbytes). + * + * However, for correct operations, it is assumed that the bitmap + * read was once serialized from a valid bitmap (i.e., it follows the format + * specification). If you provided an incorrect input (garbage), then the bitmap + * read may not be in a valid state and following operations may not lead to + * sensible results (using it may cause crashes, or it may just give incoherent + * answers). You can call roaring_bitmap_internal_validate to check the validity + * of the bitmap if the source is untrusted. Only after calling + * roaring_bitmap_internal_validate is the bitmap considered safe for use. * * The returned pointer may be NULL in case of errors. */ @@ -8494,15 +8931,20 @@ size_t roaring_bitmap_size_in_bytes(const roaring_bitmap_t *r); * * This function is unsafe in the sense that if there is no valid serialized * bitmap at the pointer, then many bytes could be read, possibly causing a - * buffer overflow. See also roaring_bitmap_portable_deserialize_safe(). + * buffer overflow. In other words, this routine assumes that `buf` points to a + * complete, correctly formatted serialized bitmap and does not take a buffer + * length argument that would let it enforce a read bound. + * + * Use this function only when the input buffer is already trusted, for example + * because it comes from memory that was previously filled by + * `roaring_bitmap_portable_serialize()` and whose size is known by some other + * means. If the source is untrusted, truncated, or otherwise not guaranteed to + * contain a valid serialized bitmap, prefer + * `roaring_bitmap_portable_deserialize_safe()`. * * This is meant to be compatible with the Java and Go versions: * https://github.com/RoaringBitmap/RoaringFormatSpec * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * The returned pointer may be NULL in case of errors. */ roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf); @@ -8536,10 +8978,6 @@ roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf); * corresponds to the serialized bitmap. The CRoaring library does not provide * checksumming. * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * The returned pointer may be NULL in case of errors. */ roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, @@ -8561,9 +8999,11 @@ roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, * This is meant to be compatible with the Java and Go versions: * https://github.com/RoaringBitmap/RoaringFormatSpec * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * Returns NULL on a big-endian system (e.g., a mainframe IBM s390x). The + * portable format is little-endian on every host, and this function uses the + * container payloads where they sit rather than converting them, so there is + * no correct in-place view of them there. Use + * `roaring_bitmap_portable_deserialize_safe()`, which converts as it copies. * * The returned pointer may be NULL in case of errors. */ @@ -8597,10 +9037,6 @@ size_t roaring_bitmap_portable_size_in_bytes(const roaring_bitmap_t *r); * This is meant to be compatible with the Java and Go versions: * https://github.com/RoaringBitmap/RoaringFormatSpec * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * When serializing data to a file, we recommend that you also use * checksums so that, at deserialization, you can be confident * that you are recovering the correct data. @@ -8637,7 +9073,8 @@ size_t roaring_bitmap_frozen_size_in_bytes(const roaring_bitmap_t *r); * * This function is endian-sensitive. If you have a big-endian system (e.g., a * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * compatible with little-endian systems. This is not a bug, it is by design, + *since the format imitates C memory layout * * When serializing data to a file, we recommend that you also use * checksums so that, at deserialization, you can be confident @@ -8658,7 +9095,8 @@ void roaring_bitmap_frozen_serialize(const roaring_bitmap_t *r, char *buf); * * This function is endian-sensitive. If you have a big-endian system (e.g., a * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * compatible with little-endian systems. This is not a bug, it is by design, + *since the format imitates C memory layout of roaring_bitmap_t. */ const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, size_t length); @@ -8679,6 +9117,16 @@ const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, bool roaring_iterate(const roaring_bitmap_t *r, roaring_iterator iterator, void *ptr); +/** + * Like `roaring_iterate`, but the 32-bit values are widened to 64 bits by + * adding `high_bits` (shifted into the upper 32 bits) before being passed to + * the iterator. This is used to build 64-bit iteration on top of 32-bit + * bitmaps. `ptr` (can be NULL) is forwarded as the second argument of each + * call. + * + * Returns true if the iterator returned true throughout (so that all values + * were necessarily visited). + */ bool roaring_iterate64(const roaring_bitmap_t *r, roaring_iterator64 iterator, uint64_t high_bits, void *ptr); @@ -9051,6 +9499,23 @@ CROARING_DEPRECATED static inline uint32_t roaring_read_uint32_iterator( return roaring_uint32_iterator_read(it, buf, count); } +/** + * Reads previous ${count} values from iterator into user-supplied ${buf}. + * Returns the number of read elements. + * This number can be smaller than ${count}, which means that iterator is + * drained. + * + * Values are written in descending order: buf[0] is the highest (current) + * value, buf[ret-1] is the lowest value read. + * + * This function satisfies semantics of reverse iteration and can be used + * together with other iterator functions. + * - first value is copied from ${it}->current_value + * - after function returns, iterator is positioned at the previous element + */ +uint32_t roaring_uint32_iterator_read_backward(roaring_uint32_iterator_t *it, + uint32_t *buf, uint32_t count); + /** * Skip the next ${count} values from iterator. * Returns the number of values actually skipped. @@ -9075,6 +9540,60 @@ uint32_t roaring_uint32_iterator_skip(roaring_uint32_iterator_t *it, uint32_t roaring_uint32_iterator_skip_backward(roaring_uint32_iterator_t *it, uint32_t count); +typedef struct roaring_uint32_range_closed_s { + uint32_t min; + uint32_t max; +} roaring_uint32_range_closed_t; + +/** + * Reads next ${count} ranges from iterator into user-supplied ${buf}. + * A range is defined as a maximal interval of consecutive values. + * For example, the set {1,2,3,5,6} contains two ranges: [1..3] and [5..6]. + * Each range is represented as a struct {min,max}, both endpoints included. + * Consecutive values that span internal container boundaries are merged into + * a single range. + * + * Returns the number of read ranges. + * This number can be smaller than ${count}, which means that the iterator is + * drained. + * + * This function satisfies the semantics of iteration and can be used together + * with other iterator functions. + * - first range will start with ${it}->current_value + * - after the function returns, the iterator is positioned at the next element + * after the end of the last returned range, or ${it}->has_value is false if + * the bitmap is exhausted. + */ +size_t roaring_uint32_iterator_read_ranges(roaring_uint32_iterator_t *it, + roaring_uint32_range_closed_t *buf, + size_t count); + +/** + * Reads previous ${count} ranges from iterator into user-supplied ${buf}. + * A range is defined as a maximal interval of consecutive values. + * For example, the set {1,2,3,5,6} contains two ranges: [1..3] and [5..6]. + * Each range is represented as a struct {min,max}, both endpoints included. + * Consecutive values that span internal container boundaries are merged into + * a single range. + * + * Returns the number of read ranges. + * This number can be smaller than ${count}, which means that the iterator is + * drained. + * + * Ranges are returned in reverse order, e.g. the first range returned is the + * highest range (ending at the current value) + * + * This function satisfies the semantics of reverse iteration and can be used + * together with other iterator functions. + * - first range will end with ${it}->current_value + * - after the function returns, the iterator is positioned at the element + * before the beginning of the last returned range, or ${it}->has_value is + * false if the bitmap is exhausted. + */ +size_t roaring_uint32_iterator_read_prev_ranges( + roaring_uint32_iterator_t *it, roaring_uint32_range_closed_t *buf, + size_t count); + #ifdef __cplusplus } } @@ -9105,6 +9624,18 @@ using namespace ::roaring::api; // in addition to roaring.h. /* end file include/roaring/roaring.h */ /* begin file include/roaring/memory.h */ +/* + * memory.h + * + * This header defines CRoaring's memory-allocation abstraction layer. It + * declares the function pointer types and hook structure used to override the + * library's malloc/realloc/calloc/free and aligned allocation routines, along + * with the wrapper functions used throughout the codebase. + * + * This allows applications to integrate CRoaring with custom allocators, + * memory trackers, arenas, or platform-specific aligned allocation policies + * without changing the rest of the library code. + */ #ifndef INCLUDE_ROARING_MEMORY_H_ #define INCLUDE_ROARING_MEMORY_H_ @@ -9146,6 +9677,15 @@ void roaring_aligned_free(void*); #endif // INCLUDE_ROARING_MEMORY_H_ /* end file include/roaring/memory.h */ /* begin file include/roaring/roaring64.h */ +/* + * roaring64.h + * + * This file declares the 64-bit Roaring bitmap API. A roaring64 bitmap stores + * sets of 64-bit unsigned integers by partitioning the value space by high + * bits and using Roaring containers for the lower bits inside each partition. + * This keeps the structure compact while preserving fast membership tests, + * insertions, iteration, and set operations over large sparse integer sets. + */ #ifndef ROARING64_H #define ROARING64_H @@ -9160,10 +9700,25 @@ namespace roaring { namespace api { #endif +/** An opaque 64-bit Roaring bitmap. Create one with `roaring64_bitmap_create()` + * and release it with `roaring64_bitmap_free()`. */ typedef struct roaring64_bitmap_s roaring64_bitmap_t; +/** Internal leaf type, exposed only for use inside `roaring64_bulk_context_t`. + * Callers should treat it as opaque. */ typedef uint64_t roaring64_leaf_t; +/** An opaque iterator over a 64-bit bitmap. See `roaring64_iterator_create()`. + */ typedef struct roaring64_iterator_s roaring64_iterator_t; +/** The leading members of `roaring64_iterator_t`, so that + * `roaring64_iterator_value()` and `roaring64_iterator_has_value()` can be + * read without a call. The iterator itself stays opaque; do not declare one of + * these, and do not rely on the layout beyond these two members. */ +typedef struct roaring64_iterator_public_s { + uint64_t value; + bool has_value; +} roaring64_iterator_public_t; + /** * A bit of context usable with `roaring64_bitmap_*_bulk()` functions. * @@ -9194,6 +9749,18 @@ void roaring64_bitmap_free(roaring64_bitmap_t *r); */ roaring64_bitmap_t *roaring64_bitmap_copy(const roaring64_bitmap_t *r); +/** + * Copies a bitmap from src to dest. It is assumed that the pointer dest + * is to an already allocated bitmap. The content of the dest bitmap is + * freed/deleted. + * + * It might be preferable and simpler to call roaring64_bitmap_copy except + * that roaring64_bitmap_overwrite can save on memory allocations. + * + */ +void roaring64_bitmap_overwrite(roaring64_bitmap_t *dest, + const roaring64_bitmap_t *src); + /** * Creates a new bitmap of a pointer to N 64-bit integers. */ @@ -9374,6 +9941,12 @@ bool roaring64_bitmap_contains(const roaring64_bitmap_t *r, uint64_t val); bool roaring64_bitmap_contains_range(const roaring64_bitmap_t *r, uint64_t min, uint64_t max); +/** + * Returns true if all values in the range [min, max] are present. + */ +bool roaring64_bitmap_contains_range_closed(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max); + /** * Check if an item is present using context from a previous insert or search * for faster search. @@ -9454,6 +10027,12 @@ uint64_t roaring64_bitmap_minimum(const roaring64_bitmap_t *r); */ uint64_t roaring64_bitmap_maximum(const roaring64_bitmap_t *r); +/** + * Remove run-length encoding even when it is more space efficient. + * Return whether a change was applied. + */ +bool roaring64_bitmap_remove_run_compression(roaring64_bitmap_t *r); + /** * Returns true if the result has at least one run container. */ @@ -9650,6 +10229,38 @@ void roaring64_bitmap_flip_inplace(roaring64_bitmap_t *r, uint64_t min, */ void roaring64_bitmap_flip_closed_inplace(roaring64_bitmap_t *r, uint64_t min, uint64_t max); +/** + * Return a copy of the bitmap with all values shifted by offset. + * + * If `positive` is true, the shift is added, otherwise subtracted. Values that + * overflow or underflow uint64_t are dropped. The caller is responsible for + * freeing the returned bitmap. + */ +roaring64_bitmap_t *roaring64_bitmap_add_offset_signed( + const roaring64_bitmap_t *r, bool positive, uint64_t offset); + +/** + * Return a copy of the bitmap with all values shifted up by offset. + * + * Values that overflow or underflow uint64_t are dropped. The caller is + * responsible for freeing the returned bitmap. + */ +static inline roaring64_bitmap_t *roaring64_bitmap_add_offset( + const roaring64_bitmap_t *r, uint64_t offset) { + return roaring64_bitmap_add_offset_signed(r, true, offset); +} + +/** + * Return a copy of the bitmap with all values shifted down by offset. + * + * Values that overflow or underflow uint64_t are dropped. The caller is + * responsible for freeing the returned bitmap. + */ +static inline roaring64_bitmap_t *roaring64_bitmap_sub_offset( + const roaring64_bitmap_t *r, uint64_t offset) { + return roaring64_bitmap_add_offset_signed(r, false, offset); +} + /** * How many bytes are required to serialize this bitmap. * @@ -9668,10 +10279,6 @@ size_t roaring64_bitmap_portable_size_in_bytes(const roaring64_bitmap_t *r); * This is meant to be compatible with other languages: * https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * When serializing data to a file, we recommend that you also use * checksums so that, at deserialization, you can be confident * that you are recovering the correct data. @@ -9716,14 +10323,50 @@ size_t roaring64_bitmap_portable_deserialize_size(const char *buf, * We also recommend that you use checksums to check that serialized data * corresponds to the serialized bitmap. The CRoaring library does not provide * checksumming. - * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. */ roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe(const char *buf, size_t maxbytes); +/** + * Read a bitmap from a portable serialized buffer as a read-only view of the + * container payloads. Headers and the ART index are allocated; bitset/array/run + * payloads alias `buf` and are not copied. + * + * In case of failure, NULL is returned. The function will not read beyond + * `maxbytes`. + * + * The returned bitmap must only be used in a readonly manner. It must be + * freed with `roaring64_bitmap_free()`. The backing buffer must outlive the + * bitmap and must not be freed or modified while it backs it. Calling any + * mutating function on the result is undefined behavior: its container array + * and headers live in a single allocation, so growing it would reallocate an + * interior pointer. + * + * The function itself is safe in the sense that it will not read beyond + * (buf, maxbytes). However, as with + * `roaring64_bitmap_portable_deserialize_safe()`, a bitmap read from garbage + * may not be in a valid state, and subsequent operations on it may not lead + * to sensible results: array containers must be sorted, and run containers + * sorted and non-overlapping, which is guaranteed only when the input came + * from a real serialized bitmap. + * + * If the source is untrusted, you should call + * `roaring64_bitmap_internal_validate` on the result before using it. Only + * after that is the bitmap considered safe for use. We also recommend + * checksumming the serialized data; CRoaring does not provide checksumming. + * + * Returns NULL on a big-endian system (e.g., a mainframe IBM s390x). The + * portable format is little-endian and this function uses the payload bytes + * where they sit, so there is no correct in-place view of them there; use + * `roaring64_bitmap_portable_deserialize_safe()`, which converts as it copies. + * + * Container payloads are used where they sit in the buffer, so they may be + * unaligned. Every access path is either SIMD with unaligned loads or marked + * `CROARING_ALLOW_UNALIGNED`. + */ +roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_frozen( + const char *buf, size_t maxbytes); + /** * Returns the number of bytes required to serialize this bitmap in a "frozen" * format. This is not compatible with any other serialization formats. @@ -9748,7 +10391,8 @@ size_t roaring64_bitmap_frozen_size_in_bytes(const roaring64_bitmap_t *r); * * This function is endian-sensitive. If you have a big-endian system (e.g., a * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * compatible with little-endian systems. This is not a bug, it is by design, + * since the format imitates C memory layout of roaring64_bitmap_t. */ size_t roaring64_bitmap_frozen_serialize(const roaring64_bitmap_t *r, char *buf); @@ -9766,7 +10410,8 @@ size_t roaring64_bitmap_frozen_serialize(const roaring64_bitmap_t *r, * * This function is endian-sensitive. If you have a big-endian system (e.g., a * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * compatible with little-endian systems. This is not a bug, it is by design, + * since the format imitates C memory layout of roaring64_bitmap_t. */ roaring64_bitmap_t *roaring64_bitmap_frozen_view(const char *buf, size_t maxbytes); @@ -9847,14 +10492,22 @@ void roaring64_iterator_free(roaring64_iterator_t *it); /** * Returns true if the iterator currently points to a value. If so, calling * `roaring64_iterator_value()` returns the value. + * + * A pointer to a structure, suitably converted, points to its initial member + * (C17 6.7.2.1p15), and `roaring64_iterator_public_t` is the initial member of + * `roaring64_iterator_t`, so this reads the field directly. */ -bool roaring64_iterator_has_value(const roaring64_iterator_t *it); +inline bool roaring64_iterator_has_value(const roaring64_iterator_t *it) { + return ((const roaring64_iterator_public_t *)it)->has_value; +} /** * Returns the value the iterator currently points to. Should only be called if * `roaring64_iterator_has_value()` returns true. */ -uint64_t roaring64_iterator_value(const roaring64_iterator_t *it); +inline uint64_t roaring64_iterator_value(const roaring64_iterator_t *it) { + return ((const roaring64_iterator_public_t *)it)->value; +} /** * Advance the iterator. If there is a new value, then @@ -9898,6 +10551,75 @@ bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, uint64_t roaring64_iterator_read(roaring64_iterator_t *it, uint64_t *buf, uint64_t count); +/** + * Reads previous ${count} values from iterator into user-supplied ${buf}. + * Returns the number of read elements. + * This number can be smaller than ${count}, which means that iterator is + * drained. + * + * Values are written in descending order: buf[0] is the highest (current) + * value, buf[ret-1] is the lowest value read. + * + * This function satisfies semantics of reverse iteration and can be used + * together with other iterator functions. + * - first value is copied from the current iterator value + * - after function returns, iterator is positioned at the previous element + */ +uint64_t roaring64_iterator_read_backward(roaring64_iterator_t *it, + uint64_t *buf, uint64_t count); + +typedef struct roaring64_range_closed_s { + uint64_t min; + uint64_t max; +} roaring64_range_closed_t; + +/** + * Reads next ${count} ranges from iterator into user-supplied ${buf}. + * A range is defined as a maximal interval of consecutive values. + * For example, the set {1,2,3,5,6} contains two ranges: [1..3] and [5..6]. + * Each range is represented as a struct {min,max}, both endpoints included. + * Consecutive values that span internal container boundaries are merged into + * a single range. + * + * Returns the number of read ranges. + * This number can be smaller than ${count}, which means that the iterator is + * drained. + * + * This function can be used together with other iterator functions. + * - first range will start with the current iterator value + * - after the function returns, the iterator is positioned at the next element + * after the end of the last returned range, or has_value is false if + * the bitmap is exhausted. + */ +size_t roaring64_iterator_read_ranges(roaring64_iterator_t *it, + roaring64_range_closed_t *buf, + size_t count); + +/** + * Reads previous ${count} ranges from iterator into user-supplied ${buf}. + * A range is defined as a maximal interval of consecutive values. + * For example, the set {1,2,3,5,6} contains two ranges: [1..3] and [5..6]. + * Each range is represented as a struct {min,max}, both endpoints included. + * Consecutive values that span internal container boundaries are merged into + * a single range. + * + * Returns the number of read ranges. + * This number can be smaller than ${count}, which means that the iterator is + * drained. + * + * Ranges are returned in reverse order, e.g. the first range returned is the + * highest range (ending at the current value). + * + * This function can be used together with other iterator functions. + * - first range will end with the current iterator value + * - after the function returns, the iterator is positioned at the element + * before the beginning of the last returned range, or has_value is false if + * the bitmap is exhausted. + */ +size_t roaring64_iterator_read_prev_ranges(roaring64_iterator_t *it, + roaring64_range_closed_t *buf, + size_t count); + #ifdef __cplusplus } // extern "C" } // namespace roaring diff --git a/Sources/CRoaring/roaring.c b/Sources/CRoaring/roaring.c index 56e04f2..f753a21 100644 --- a/Sources/CRoaring/roaring.c +++ b/Sources/CRoaring/roaring.c @@ -1,5 +1,5 @@ // !!! DO NOT EDIT - THIS IS AN AUTO-GENERATED FILE !!! -// Created by amalgamation.sh on 2025-12-05T04:42:38Z +// Created by amalgamation.sh on 2026-08-22T02:17:56Z /* * The CRoaring project is under a dual license (Apache/MIT). @@ -63,6 +63,17 @@ #include "roaring.h" /* include public API definitions */ /* begin file include/roaring/containers/perfparameters.h */ +/* + * perfparameters.h + * + * This header centralizes a small set of performance-tuning constants and + * heuristic defaults used by CRoaring container code. These parameters control + * decisions such as initial container sizing and when lazy or eager operations + * may convert between container representations. + * + * In practice, these values encode trade-offs between memory use, allocation + * overhead, and execution speed for common workloads. + */ #ifndef PERFPARAMETERS_H_ #define PERFPARAMETERS_H_ @@ -117,6 +128,15 @@ enum { ARRAY_DEFAULT_INIT_SIZE = 0 }; /* * utilasm.h * + * This file provides optional inline-assembly helpers for low-level bit + * manipulation on supported x86/x64 targets. These macros are used to map a + * few performance-sensitive operations, such as shifting, testing, setting, + * and clearing bits, to specific machine instructions when inline assembly is + * enabled. + * + * The intent is to centralize these architecture-specific primitives behind a + * small interface so the rest of the codebase can use them conditionally while + * keeping the generic implementation paths separate. */ #ifndef INCLUDE_UTILASM_H_ @@ -819,9 +839,8 @@ static const uint8_t shuffle_mask16[] = { * Optimized by D. Lemire on May 3rd 2013 */ CROARING_TARGET_AVX2 -int32_t intersect_vector16(const uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b, - uint16_t *C) { +int32_t intersect_vector16(const uint16_t *A, size_t s_a, const uint16_t *B, + size_t s_b, uint16_t *C) { size_t count = 0; size_t i_a = 0, i_b = 0; const int vectorlength = sizeof(__m128i) / sizeof(uint16_t); @@ -920,8 +939,8 @@ int array_container_to_uint32_array_vector16(void *vout, const uint16_t *array, return outpos; } -int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b) { +int32_t intersect_vector16_inplace(uint16_t *A, size_t s_a, const uint16_t *B, + size_t s_b) { size_t count = 0; size_t i_a = 0, i_b = 0; const int vectorlength = sizeof(__m128i) / sizeof(uint16_t); @@ -1015,10 +1034,8 @@ int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -int32_t intersect_vector16_cardinality(const uint16_t *__restrict__ A, - size_t s_a, - const uint16_t *__restrict__ B, - size_t s_b) { +int32_t intersect_vector16_cardinality(const uint16_t *A, size_t s_a, + const uint16_t *B, size_t s_b) { size_t count = 0; size_t i_a = 0, i_b = 0; const int vectorlength = sizeof(__m128i) / sizeof(uint16_t); @@ -1091,9 +1108,8 @@ CROARING_TARGET_AVX2 // Warning: // This function may not be safe if A == C or B == C. ///////// -int32_t difference_vector16(const uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b, - uint16_t *C) { +int32_t difference_vector16(const uint16_t *A, size_t s_a, const uint16_t *B, + size_t s_b, uint16_t *C) { // we handle the degenerate case if (s_a == 0) return 0; if (s_b == 0) { @@ -2082,17 +2098,27 @@ static inline uint32_t unique(uint16_t *out, uint32_t len) { return pos; } -// use with qsort, could be avoided -static int uint16_compare(const void *a, const void *b) { - return (*(uint16_t *)a - *(uint16_t *)b); +// Sort a very short run of uint16 values in place. The callers below feed this +// at most 16 values; calling qsort() for that costs more in glibc merge-sort +// setup, function-pointer compares and memmove traffic than the sort itself. +static inline void sort_uint16_short(uint16_t *z, uint32_t n) { + for (uint32_t i = 1; i < n; i++) { + uint16_t v = z[i]; + uint32_t j = i; + while (j > 0 && z[j - 1] > v) { + z[j] = z[j - 1]; + j--; + } + z[j] = v; + } } CROARING_TARGET_AVX2 // a one-pass SSE union algorithm // This function may not be safe if array1 == output or array2 == output. -uint32_t union_vector16(const uint16_t *__restrict__ array1, uint32_t length1, - const uint16_t *__restrict__ array2, uint32_t length2, - uint16_t *__restrict__ output) { +uint32_t union_vector16(const uint16_t *array1, uint32_t length1, + const uint16_t *array2, uint32_t length2, + uint16_t *output) { if ((length1 < 8) || (length2 < 8)) { return (uint32_t)union_uint16(array1, length1, array2, length2, output); } @@ -2153,7 +2179,7 @@ uint32_t union_vector16(const uint16_t *__restrict__ array1, uint32_t length1, memcpy(buffer + leftoversize, array1 + 8 * pos1, (length1 - 8 * len1) * sizeof(uint16_t)); leftoversize += length1 - 8 * len1; - qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + sort_uint16_short(buffer, leftoversize); leftoversize = unique(buffer, leftoversize); len += (uint32_t)union_uint16(buffer, leftoversize, array2 + 8 * pos2, @@ -2162,7 +2188,7 @@ uint32_t union_vector16(const uint16_t *__restrict__ array1, uint32_t length1, memcpy(buffer + leftoversize, array2 + 8 * pos2, (length2 - 8 * len2) * sizeof(uint16_t)); leftoversize += length2 - 8 * len2; - qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + sort_uint16_short(buffer, leftoversize); leftoversize = unique(buffer, leftoversize); len += (uint32_t)union_uint16(buffer, leftoversize, array1 + 8 * pos1, length1 - 8 * pos1, output); @@ -2176,6 +2202,193 @@ CROARING_UNTARGET_AVX2 * */ +/** + * Start of the AVX-512 16-bit union code. + * + * union_vector16 above merges 8 lanes at a time with an odd-even transposition + * network: 8 dependent min/max stages per 8 output values, which measures at + * ~2.3 cycles per input element -- no better than the scalar union_uint16. + * + * With AVX-512 we can merge 32+32 lanes with a Batcher bitonic network: one + * reverse plus 5 compare-exchange stages per sorted half, i.e. ~4x fewer + * operations per output value. This needs cross-lane 16-bit permutes (vpermw, + * vpermt2w) and a 16-bit compress store (vpcompressw); AVX2 has none of these, + * so the algorithm is genuinely AVX-512-only rather than a wider rerun of the + * SSE code. + */ +#if CROARING_COMPILER_SUPPORTS_AVX512 + +CROARING_TARGET_AVX512 + +// A compare-exchange at distance d pairs lane i with lane i^d and keeps the +// smaller value in whichever lane has its d-bit clear. `hi` selects the lanes +// whose d-bit is set. +static inline __m512i avx512_cx16(__m512i v, __m512i t, __mmask32 hi) { + return _mm512_mask_mov_epi16(_mm512_min_epu16(v, t), hi, + _mm512_max_epu16(v, t)); +} + +// Sort a bitonic 32-lane sequence into ascending order. Each distance has a +// dedicated cheap shuffle, so no stage needs a full vpermw. +static inline __m512i avx512_bitonic_sort32(__m512i v) { + v = avx512_cx16(v, _mm512_shuffle_i64x2(v, v, 0x4E), 0xFFFF0000u); // d=16 + v = avx512_cx16(v, _mm512_shuffle_i64x2(v, v, 0xB1), 0xFF00FF00u); // d=8 + v = avx512_cx16(v, _mm512_shuffle_epi32(v, 0x4E), 0xF0F0F0F0u); // d=4 + v = avx512_cx16(v, _mm512_shuffle_epi32(v, 0xB1), 0xCCCCCCCCu); // d=2 + v = avx512_cx16(v, _mm512_rol_epi32(v, 16), 0xAAAAAAAAu); // d=1 + return v; +} + +// Merge two ascending 32-lane vectors: *lo receives the 32 smallest values in +// ascending order, *hi the 32 largest, also ascending. +static inline void avx512_bitonic_merge32(__m512i a, __m512i b, __m512i *lo, + __m512i *hi) { + static const uint16_t revtab[32] = { + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + __m512i br = _mm512_permutexvar_epi16( + _mm512_loadu_si512((const __m512i *)revtab), b); + *lo = avx512_bitonic_sort32(_mm512_min_epu16(a, br)); + *hi = avx512_bitonic_sort32(_mm512_max_epu16(a, br)); +} + +// Write the values of the ascending vector `v` that differ from their +// predecessor, where the predecessor of lane 0 is *last. Updates *last to the +// largest value emitted and returns how many values were written. +static inline int avx512_emit_unique16(__m512i v, uint16_t *out, + uint16_t *last) { + static const uint16_t shift1[32] = { + 32, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}; + __m512i prev = _mm512_permutex2var_epi16( + v, _mm512_loadu_si512((const __m512i *)shift1), + _mm512_set1_epi16((short)*last)); + __mmask32 keep = _mm512_cmpneq_epi16_mask(v, prev); + _mm512_mask_compressstoreu_epi16(out, keep, v); + *last = (uint16_t)_mm_extract_epi16(_mm512_extracti32x4_epi32(v, 3), 7); + return (int)roaring_hamming(keep); +} + +/** + * A one-pass AVX-512 union of two sorted uint16 arrays. + * + * As with union_vector16, the caller must guarantee that `output` has room for + * size_1 + size_2 values. Partial overlap of `output` with the inputs is + * tolerated in exactly the same way union_vector16 tolerates it (see the long + * comment in array_run_container_inplace_union): after consuming p1 + p2 whole + * 32-lane blocks this routine has written at most 32*(p1+p2) - 32 values, + * because one merged block is always held back in `vmax`. The output pointer + * therefore stays strictly behind the read pointers. + * + * That bound is exactly tight rather than merely satisfied: in the worst case + * the final merge below writes its last value to precisely the slot the last + * input value was read from. Anything that reduces how much the tail holds back + * -- shrinking `pending`, emitting `vmax` sooner -- silently breaks aliased + * callers, so re-derive the bound before changing the buffering here. + */ +uint32_t avx512_union_uint16(const uint16_t *array1, uint32_t length1, + const uint16_t *array2, uint32_t length2, + uint16_t *output) { + const uint32_t W = 32; // lanes per 512-bit register + if (length1 < W || length2 < W) { + // Not enough on one side to fill a block. union_vector16 keeps + // vectorizing at 8-lane granularity, so it still beats the scalar + // merge once there is a worthwhile amount of data on the other side; + // below that its own fixed overhead dominates. + if (length1 + length2 >= 64) { + return union_vector16(array1, length1, array2, length2, output); + } + return (uint32_t)union_uint16(array1, length1, array2, length2, output); + } + const uint32_t blocks1 = length1 / W, blocks2 = length2 / W; + uint32_t p1 = 0, p2 = 0; + uint16_t *out = output; + __m512i vmin, vmax; + + avx512_bitonic_merge32(_mm512_loadu_si512((const __m512i *)array1), + _mm512_loadu_si512((const __m512i *)array2), &vmin, + &vmax); + p1 = 1; + p2 = 1; + // Lane 0 of the very first block has no predecessor. Seeding `last` with a + // value that differs from it in one bit keeps the first value. + uint16_t last = + (uint16_t)(_mm_extract_epi16(_mm512_castsi512_si128(vmin), 0) ^ 1); + out += avx512_emit_unique16(vmin, out, &last); + + while (p1 < blocks1 && p2 < blocks2) { + // Which side advances is essentially unpredictable, so select the + // source pointer without branching. + const uint16_t *pa = array1 + W * p1; + const uint16_t *pb = array2 + W * p2; + const uint32_t take1 = (*pa <= *pb) ? 1 : 0; + const __m512i v = + _mm512_loadu_si512((const __m512i *)(take1 ? pa : pb)); + p1 += take1; + p2 += 1 - take1; + avx512_bitonic_merge32(v, vmax, &vmin, &vmax); + out += avx512_emit_unique16(vmin, out, &last); + } + + // The ragged tail: the values still held in vmax, plus at most W-1 leftover + // values from the exhausted side, plus the rest of the other side. Two + // two-way merges branch-predict far better than one three-way merge. + // + // vmax can hold the same value twice (once from each input), so it goes + // through avx512_emit_unique16 rather than a raw store: that dedups within + // the block and against `last` at the same time. Every remaining tail value + // is >= last and the tail is sorted, so for the two scalar inputs only the + // very first value can still duplicate `last`. + uint16_t pending[32]; + uint16_t pending_last = last; + size_t npending = + (size_t)avx512_emit_unique16(vmax, pending, &pending_last); + + const uint16_t *rest1 = array1 + W * p1, *rest2 = array2 + W * p2; + size_t nrest1 = length1 - W * p1, nrest2 = length2 - W * p2; + const uint16_t *shortrest, *longrest; + size_t nshort, nlong; + if (p1 == blocks1) { + shortrest = rest1; + nshort = nrest1; + longrest = rest2; + nlong = nrest2; + } else { + shortrest = rest2; + nshort = nrest2; + longrest = rest1; + nlong = nrest1; + } + if (nshort > 0 && shortrest[0] == last) { + shortrest++; + nshort--; + } + if (nlong > 0 && longrest[0] == last) { + longrest++; + nlong--; + } + + uint16_t merged[2 * 32]; // <= W pending + (W-1) leftover + size_t nmerged = union_uint16(pending, npending, shortrest, nshort, merged); + // When the small side held fewer than two blocks the loop above ran at most + // once, so `longrest` can still be most of the larger input. Finishing that + // scalar would give away more than the block loop won, hence the same + // 8-lane handoff as in the short-input case. + if (nlong >= 64) { + out += union_vector16(merged, (uint32_t)nmerged, longrest, + (uint32_t)nlong, out); + } else { + out += union_uint16(merged, nmerged, longrest, nlong, out); + } + return (uint32_t)(out - output); +} +CROARING_UNTARGET_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + +/** + * End of the AVX-512 16-bit union code. + */ + /** * Start of SIMD 16-bit XOR code */ @@ -2214,9 +2427,9 @@ static inline uint32_t unique_xor(uint16_t *out, uint32_t len) { } CROARING_TARGET_AVX2 // a one-pass SSE xor algorithm -uint32_t xor_vector16(const uint16_t *__restrict__ array1, uint32_t length1, - const uint16_t *__restrict__ array2, uint32_t length2, - uint16_t *__restrict__ output) { +uint32_t xor_vector16(const uint16_t *array1, uint32_t length1, + const uint16_t *array2, uint32_t length2, + uint16_t *output) { if ((length1 < 8) || (length2 < 8)) { return xor_uint16(array1, length1, array2, length2, output); } @@ -2294,7 +2507,7 @@ uint32_t xor_vector16(const uint16_t *__restrict__ array1, uint32_t length1, (length2 - 8 * pos2) * sizeof(uint16_t)); len += (length2 - 8 * pos2); } else { - qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + sort_uint16_short(buffer, leftoversize); leftoversize = unique_xor(buffer, leftoversize); len += xor_uint16(buffer, leftoversize, array2 + 8 * pos2, length2 - 8 * pos2, output); @@ -2308,7 +2521,7 @@ uint32_t xor_vector16(const uint16_t *__restrict__ array1, uint32_t length1, (length1 - 8 * pos1) * sizeof(uint16_t)); len += (length1 - 8 * pos1); } else { - qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + sort_uint16_short(buffer, leftoversize); leftoversize = unique_xor(buffer, leftoversize); len += xor_uint16(buffer, leftoversize, array1 + 8 * pos1, length1 - 8 * pos1, output); @@ -2420,7 +2633,20 @@ size_t fast_union_uint16(const uint16_t *set_1, size_t size_1, const uint16_t *set_2, size_t size_2, uint16_t *buffer) { #if CROARING_IS_X64 - if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + const unsigned support = (unsigned)croaring_hardware_support(); +#if CROARING_COMPILER_SUPPORTS_AVX512 + if (support & ROARING_SUPPORTS_AVX512) { + // compute union with smallest array first + if (size_1 < size_2) { + return avx512_union_uint16(set_1, (uint32_t)size_1, set_2, + (uint32_t)size_2, buffer); + } else { + return avx512_union_uint16(set_2, (uint32_t)size_2, set_1, + (uint32_t)size_1, buffer); + } + } +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + if (support & ROARING_SUPPORTS_AVX2) { // compute union with smallest array first if (size_1 < size_2) { return union_vector16(set_1, (uint32_t)size_1, set_2, @@ -2609,7 +2835,6 @@ CROARING_UNTARGET_AVX512 #endif/* end file src/array_util.c */ /* begin file src/art/art.c */ #include -#include #include #include @@ -4150,7 +4375,7 @@ static void art_node_print_type(art_ref_t ref) { } } -void art_node_printf(const art_t *art, art_ref_t ref, uint8_t depth) { +static void art_node_printf(const art_t *art, art_ref_t ref, uint8_t depth) { if (art_is_leaf(ref)) { printf("{ type: Leaf, key: "); art_leaf_t *leaf = (art_leaf_t *)art_deref(art, ref); @@ -5584,6 +5809,7 @@ const uint8_t vbmi2_table[64] = { size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, uint32_t *vout, size_t outcapacity, uint32_t base) { + if (outcapacity == 0) return 0; uint32_t *out = (uint32_t *)vout; uint32_t *initout = out; uint32_t *safeout = out + outcapacity; @@ -5640,6 +5866,7 @@ size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, size_t bitset_extract_setbits_avx512_uint16(const uint64_t *array, size_t length, uint16_t *vout, size_t capacity, uint16_t base) { + if (capacity == 0) return 0; uint16_t *out = (uint16_t *)vout; uint16_t *initout = out; uint16_t *safeout = vout + capacity; @@ -5691,6 +5918,7 @@ CROARING_TARGET_AVX2 size_t bitset_extract_setbits_avx2(const uint64_t *words, size_t length, uint32_t *out, size_t outcapacity, uint32_t base) { + if (outcapacity == 0) return 0; uint32_t *initout = out; __m256i baseVec = _mm256_set1_epi32(base - 1); __m256i incVec = _mm256_set1_epi32(64); @@ -5794,6 +6022,7 @@ CROARING_TARGET_AVX2 size_t bitset_extract_setbits_sse_uint16(const uint64_t *words, size_t length, uint16_t *out, size_t outcapacity, uint16_t base) { + if (outcapacity == 0) return 0; uint16_t *initout = out; __m128i baseVec = _mm_set1_epi16(base - 1); __m128i incVec = _mm_set1_epi16(64); @@ -6755,17 +6984,19 @@ void array_container_offset(const array_container_t *c, container_t **loc, if (loc && lo_cap) { lo = array_container_create_given_capacity(lo_cap); for (int i = 0; i < lo_cap; ++i) { - array_container_add(lo, c->array[i] + offset); + lo->array[i] = c->array[i] + offset; } + lo->cardinality = lo_cap; *loc = (container_t *)lo; } hi_cap = c->cardinality - lo_cap; if (hic && hi_cap) { hi = array_container_create_given_capacity(hi_cap); - for (int i = lo_cap; i < c->cardinality; ++i) { - array_container_add(hi, c->array[i] + offset); + for (int i = 0; i < hi_cap; ++i) { + hi->array[i] = c->array[lo_cap + i] + offset; } + hi->cardinality = hi_cap; *hic = (container_t *)hi; } } @@ -7152,7 +7383,14 @@ int32_t array_container_number_of_runs(const array_container_t *ac) { * */ int32_t array_container_write(const array_container_t *container, char *buf) { +#if CROARING_IS_BIG_ENDIAN + for (int32_t i = 0; i < container->cardinality; ++i) { + uint16_t v_le = croaring_htole16(container->array[i]); + memcpy(buf + i * sizeof(uint16_t), &v_le, sizeof(uint16_t)); + } +#else memcpy(buf, container->array, container->cardinality * sizeof(uint16_t)); +#endif return array_container_size_in_bytes(container); } @@ -7185,7 +7423,15 @@ int32_t array_container_read(int32_t cardinality, array_container_t *container, array_container_grow(container, cardinality, false); } container->cardinality = cardinality; +#if CROARING_IS_BIG_ENDIAN + for (int32_t i = 0; i < cardinality; ++i) { + uint16_t v_le; + memcpy(&v_le, buf + i * sizeof(uint16_t), sizeof(uint16_t)); + container->array[i] = croaring_letoh16(v_le); + } +#else memcpy(container->array, buf, container->cardinality * sizeof(uint16_t)); +#endif return array_container_size_in_bytes(container); } @@ -7268,8 +7514,7 @@ void bitset_container_set_all(bitset_container_t *bitset) { bitset->cardinality = (1 << 16); } -/* Create a new bitset. Return NULL in case of failure. */ -bitset_container_t *bitset_container_create(void) { +static bitset_container_t *bitset_container_allocate(void) { bitset_container_t *bitset = (bitset_container_t *)roaring_malloc(sizeof(bitset_container_t)); @@ -7294,7 +7539,23 @@ bitset_container_t *bitset_container_create(void) { roaring_free(bitset); return NULL; } - bitset_container_clear(bitset); + return bitset; +} + +/* Create a new bitset. Return NULL in case of failure. */ +bitset_container_t *bitset_container_create(void) { + bitset_container_t *bitset = bitset_container_allocate(); + if (bitset) { + bitset_container_clear(bitset); + } + return bitset; +} + +bitset_container_t *bitset_container_create_uninitialized(void) { + bitset_container_t *bitset = bitset_container_allocate(); + if (bitset) { + bitset->cardinality = 0; + } return bitset; } @@ -8257,7 +8518,14 @@ int bitset_container_number_of_runs(bitset_container_t *bc) { int32_t bitset_container_write(const bitset_container_t *container, char *buf) { +#if CROARING_IS_BIG_ENDIAN + for (int32_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; ++i) { + uint64_t w_le = croaring_htole64(container->words[i]); + memcpy(buf + i * sizeof(uint64_t), &w_le, sizeof(uint64_t)); + } +#else memcpy(buf, container->words, BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t)); +#endif return bitset_container_size_in_bytes(container); } @@ -8265,7 +8533,15 @@ int32_t bitset_container_write(const bitset_container_t *container, int32_t bitset_container_read(int32_t cardinality, bitset_container_t *container, const char *buf) { container->cardinality = cardinality; +#if CROARING_IS_BIG_ENDIAN + for (int32_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; ++i) { + uint64_t w_le; + memcpy(&w_le, buf + i * sizeof(uint64_t), sizeof(uint64_t)); + container->words[i] = croaring_letoh64(w_le); + } +#else memcpy(container->words, buf, BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t)); +#endif return bitset_container_size_in_bytes(container); } @@ -8561,6 +8837,10 @@ extern bool container_iterator_next(const container_t *c, uint8_t typecode, extern bool container_iterator_prev(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint16_t *value); +extern bool container_contains( + const container_t *c, uint16_t val, + uint8_t typecode // !!! should be second argument? +); void container_free(container_t *c, uint8_t type) { switch (type) { @@ -8806,6 +9086,7 @@ extern inline container_t *container_andnot(const container_t *c1, uint8_t type2, uint8_t *result_type); +CROARING_ALLOW_UNALIGNED roaring_container_iterator_t container_init_iterator(const container_t *c, uint8_t typecode, uint16_t *value) { @@ -8845,6 +9126,7 @@ roaring_container_iterator_t container_init_iterator(const container_t *c, } } +CROARING_ALLOW_UNALIGNED roaring_container_iterator_t container_init_iterator_last(const container_t *c, uint8_t typecode, uint16_t *value) { @@ -8924,6 +9206,7 @@ bool container_iterator_lower_bound(const container_t *c, uint8_t typecode, } } +CROARING_ALLOW_UNALIGNED bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint32_t high16, uint32_t *buf, @@ -8967,8 +9250,10 @@ bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, const array_container_t *ac = const_CAST_array(c); uint32_t num_values = minimum_uint32(ac->cardinality - it->index, count); + // Hoist so GCC can vectorize the uint16->uint32 widen-or. + const uint16_t *src = ac->array + it->index; for (uint32_t i = 0; i < num_values; i++) { - buf[i] = high16 | ac->array[it->index + i]; + buf[i] = high16 | src[i]; } *consumed += num_values; it->index += num_values; @@ -9013,6 +9298,7 @@ bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, } } +CROARING_ALLOW_UNALIGNED bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint64_t high48, uint64_t *buf, @@ -9056,8 +9342,10 @@ bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, const array_container_t *ac = const_CAST_array(c); uint32_t num_values = minimum_uint32(ac->cardinality - it->index, count); + // Hoist so GCC can vectorize the uint16->uint64 widen-or. + const uint16_t *src = ac->array + it->index; for (uint32_t i = 0; i < num_values; i++) { - buf[i] = high48 | ac->array[it->index + i]; + buf[i] = high48 | src[i]; } *consumed += num_values; it->index += num_values; @@ -9102,6 +9390,186 @@ bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, } } +CROARING_ALLOW_UNALIGNED +bool container_iterator_read_backward_into_uint32( + const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, + uint32_t high16, uint32_t *buf, uint32_t count, uint32_t *consumed, + uint16_t *value_out) { + *consumed = 0; + if (count == 0) { + return false; + } + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = it->index / 64; + uint64_t word = + bc->words[wordindex] & (UINT64_MAX >> (63 - (it->index % 64))); + do { + // Read set bits. + while (word != 0 && *consumed < count) { + uint32_t bit = 63 - roaring_leading_zeroes(word); + *buf = high16 | (wordindex * 64 + bit); + word &= ~(UINT64_C(1) << bit); + buf++; + (*consumed)++; + } + // Skip unset bits. + while (word == 0 && wordindex > 0) { + wordindex--; + word = bc->words[wordindex]; + } + } while (word != 0 && *consumed < count); + + if (word != 0) { + it->index = + wordindex * 64 + (63 - roaring_leading_zeroes(word)); + *value_out = it->index; + return true; + } + return false; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint32_t num_values = + minimum_uint32((uint32_t)(it->index + 1), count); + // Walk backwards so GCC can vectorize the uint16->uint32 widen-or. + const uint16_t *src = ac->array + it->index + 1; + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high16 | *--src; + } + *consumed += num_values; + it->index -= num_values; + if (it->index >= 0) { + *value_out = ac->array[it->index]; + return true; + } + return false; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + do { + uint32_t run_start = rc->runs[it->index].value; + uint32_t num_values = minimum_uint32(*value_out - run_start + 1, + count - *consumed); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high16 | (*value_out - i); + } + *value_out -= num_values; + buf += num_values; + *consumed += num_values; + + // We check for `value == UINT16_MAX` because + // `*value_out -= num_values` can underflow when + // `value == 0` (run_start == 0). In this case `value` + // will underflow to UINT16_MAX. + if (*value_out < run_start || *value_out == UINT16_MAX) { + it->index--; + if (it->index >= 0) { + *value_out = rc->runs[it->index].value + + rc->runs[it->index].length; + } else { + return false; + } + } + } while (*consumed < count); + return true; + } + default: + assert(false); + roaring_unreachable; + return 0; + } +} + +CROARING_ALLOW_UNALIGNED +bool container_iterator_read_backward_into_uint64( + const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, + uint64_t high48, uint64_t *buf, uint32_t count, uint32_t *consumed, + uint16_t *value_out) { + *consumed = 0; + if (count == 0) { + return false; + } + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = it->index / 64; + uint64_t word = + bc->words[wordindex] & (UINT64_MAX >> (63 - (it->index % 64))); + do { + // Read set bits. + while (word != 0 && *consumed < count) { + uint32_t bit = 63 - roaring_leading_zeroes(word); + *buf = high48 | (wordindex * 64 + bit); + word &= ~(UINT64_C(1) << bit); + buf++; + (*consumed)++; + } + // Skip unset bits. + while (word == 0 && wordindex > 0) { + wordindex--; + word = bc->words[wordindex]; + } + } while (word != 0 && *consumed < count); + + if (word != 0) { + it->index = + wordindex * 64 + (63 - roaring_leading_zeroes(word)); + *value_out = it->index; + return true; + } + return false; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint32_t num_values = + minimum_uint32((uint32_t)(it->index + 1), count); + // Walk backwards so GCC can vectorize the uint16->uint64 widen-or. + const uint16_t *src = ac->array + it->index + 1; + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high48 | *--src; + } + *consumed += num_values; + it->index -= num_values; + if (it->index >= 0) { + *value_out = ac->array[it->index]; + return true; + } + return false; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + do { + uint32_t run_start = rc->runs[it->index].value; + uint32_t num_values = minimum_uint32(*value_out - run_start + 1, + count - *consumed); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high48 | (*value_out - i); + } + *value_out -= num_values; + buf += num_values; + *consumed += num_values; + + if (*value_out < run_start || *value_out == UINT16_MAX) { + it->index--; + if (it->index >= 0) { + *value_out = rc->runs[it->index].value + + rc->runs[it->index].length; + } else { + return false; + } + } + } while (*consumed < count); + return true; + } + default: + assert(false); + roaring_unreachable; + return 0; + } +} + bool container_iterator_skip(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint32_t skip_count, uint32_t *consumed_count, @@ -9301,34 +9769,199 @@ bool container_iterator_skip_backward(const container_t *c, uint8_t typecode, return has_value; } -#ifdef __cplusplus -} +uint16_t container_iterator_find_run_end(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value, bool *has_more) { + switch (typecode) { + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + uint16_t run_end = + rc->runs[it->index].value + rc->runs[it->index].length; + it->index++; + if (it->index < rc->n_runs) { + *has_more = true; + *value = rc->runs[it->index].value; + } else { + *has_more = false; + } + return run_end; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint16_t v = *value; + while (it->index + 1 < ac->cardinality && + ac->array[it->index + 1] == (uint16_t)(v + 1)) { + it->index++; + v++; + } + it->index++; + if (it->index < ac->cardinality) { + *has_more = true; + *value = ac->array[it->index]; + } else { + *has_more = false; + } + return v; + } + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t pos = (uint32_t)*value + 1; + uint16_t run_end; + if (pos >= (1 << 16)) { + *has_more = false; + return UINT16_MAX; + } + uint32_t wordindex = pos / 64; + uint64_t word = ~bc->words[wordindex] & (UINT64_MAX << (pos % 64)); + while (word == 0 && + wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { + wordindex++; + word = ~bc->words[wordindex]; + } + if (word != 0) { + run_end = (uint16_t)(wordindex * 64 + + roaring_trailing_zeroes(word) - 1); + } else { + run_end = UINT16_MAX; + } + uint32_t next_pos = (uint32_t)run_end + 1; + if (next_pos >= (1 << 16)) { + *has_more = false; + } else { + wordindex = next_pos / 64; + word = bc->words[wordindex] & (UINT64_MAX << (next_pos % 64)); + while (word == 0 && + wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { + wordindex++; + word = bc->words[wordindex]; + } + if (word != 0) { + *has_more = true; + it->index = wordindex * 64 + roaring_trailing_zeroes(word); + *value = (uint16_t)it->index; + } else { + *has_more = false; + } + } + return run_end; + } + default: + assert(false); + roaring_unreachable; + return 0; + } } -} // extern "C" { namespace roaring { namespace internal { -#endif - -#undef ROARING_INIT_ROARING_CONTAINER_ITERATOR_T -/* end file src/containers/containers.c */ -/* begin file src/containers/convert.c */ -#include - -#if CROARING_IS_X64 -#ifndef CROARING_COMPILER_SUPPORTS_AVX512 -#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." -#endif // CROARING_COMPILER_SUPPORTS_AVX512 -#endif - -#ifdef __cplusplus -extern "C" { -namespace roaring { -namespace internal { -#endif - -// file contains grubby stuff that must know impl. details of all container -// types. -bitset_container_t *bitset_container_from_array(const array_container_t *ac) { - bitset_container_t *ans = bitset_container_create(); +uint16_t container_iterator_find_run_start(const container_t *c, + uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value, bool *has_more) { + switch (typecode) { + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + uint16_t run_start = rc->runs[it->index].value; + it->index--; + if (it->index >= 0) { + *has_more = true; + *value = rc->runs[it->index].value + rc->runs[it->index].length; + } else { + *has_more = false; + } + return run_start; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint16_t v = *value; + while (it->index > 0 && + ac->array[it->index - 1] == (uint16_t)(v - 1)) { + it->index--; + v--; + } + it->index--; + if (it->index >= 0) { + *has_more = true; + *value = ac->array[it->index]; + } else { + *has_more = false; + } + return v; + } + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + if (*value == 0) { + *has_more = false; + return 0; + } + uint32_t pos = (uint32_t)*value - 1; + int32_t wordindex = (int32_t)(pos / 64); + uint64_t word = + ~bc->words[wordindex] & (UINT64_MAX >> (63 - (pos % 64))); + while (word == 0 && --wordindex >= 0) { + word = ~bc->words[wordindex]; + } + uint16_t run_start; + if (word != 0) { + run_start = (uint16_t)(wordindex * 64 + + (63 - roaring_leading_zeroes(word)) + 1); + } else { + run_start = 0; + } + if (run_start == 0) { + *has_more = false; + } else { + int32_t prev_pos = (int32_t)run_start - 1; + wordindex = prev_pos / 64; + word = bc->words[wordindex] & + (UINT64_MAX >> (63 - (prev_pos % 64))); + while (word == 0 && --wordindex >= 0) { + word = bc->words[wordindex]; + } + if (word != 0) { + *has_more = true; + it->index = + wordindex * 64 + (63 - roaring_leading_zeroes(word)); + *value = (uint16_t)it->index; + } else { + *has_more = false; + } + } + return run_start; + } + default: + assert(false); + roaring_unreachable; + return 0; + } +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#undef ROARING_INIT_ROARING_CONTAINER_ITERATOR_T +/* end file src/containers/containers.c */ +/* begin file src/containers/convert.c */ +#include + + +#if CROARING_IS_X64 +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +// file contains grubby stuff that must know impl. details of all container +// types. +bitset_container_t *bitset_container_from_array(const array_container_t *ac) { + bitset_container_t *ans = bitset_container_create(); int limit = array_container_cardinality(ac); for (int i = 0; i < limit; ++i) bitset_container_set(ans, ac->array[i]); return ans; @@ -12581,9 +13214,21 @@ bool run_container_validate(const run_container_t *run, const char **reason) { int32_t run_container_write(const run_container_t *container, char *buf) { uint16_t cast_16 = container->n_runs; - memcpy(buf, &cast_16, sizeof(uint16_t)); + uint16_t n_runs_le = croaring_htole16(cast_16); + memcpy(buf, &n_runs_le, sizeof(uint16_t)); +#if CROARING_IS_BIG_ENDIAN + char *out = buf + sizeof(uint16_t); + for (int32_t i = 0; i < container->n_runs; ++i) { + uint16_t v_le = croaring_htole16(container->runs[i].value); + uint16_t l_le = croaring_htole16(container->runs[i].length); + memcpy(out, &v_le, sizeof(uint16_t)); + memcpy(out + sizeof(uint16_t), &l_le, sizeof(uint16_t)); + out += sizeof(rle16_t); + } +#else memcpy(buf + sizeof(uint16_t), container->runs, container->n_runs * sizeof(rle16_t)); +#endif return run_container_size_in_bytes(container); } @@ -12592,12 +13237,24 @@ int32_t run_container_read(int32_t cardinality, run_container_t *container, (void)cardinality; uint16_t cast_16; memcpy(&cast_16, buf, sizeof(uint16_t)); - container->n_runs = cast_16; + container->n_runs = croaring_letoh16(cast_16); if (container->n_runs > container->capacity) run_container_grow(container, container->n_runs, false); if (container->n_runs > 0) { +#if CROARING_IS_BIG_ENDIAN + const char *in = buf + sizeof(uint16_t); + for (int32_t i = 0; i < container->n_runs; ++i) { + uint16_t v_le, l_le; + memcpy(&v_le, in, sizeof(uint16_t)); + memcpy(&l_le, in + sizeof(uint16_t), sizeof(uint16_t)); + container->runs[i].value = croaring_letoh16(v_le); + container->runs[i].length = croaring_letoh16(l_le); + in += sizeof(rle16_t); + } +#else memcpy(container->runs, buf + sizeof(uint16_t), container->n_runs * sizeof(rle16_t)); +#endif } return run_container_size_in_bytes(container); } @@ -12792,9 +13449,11 @@ int run_container_get_index(const run_container_t *container, uint16_t x) { return -1; } } +#define CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY 0 #if defined(CROARING_IS_X64) && CROARING_COMPILER_SUPPORTS_AVX512 +#if CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY CROARING_TARGET_AVX512 CROARING_ALLOW_UNALIGNED /* Get the cardinality of `run'. Requires an actual computation. */ @@ -12821,6 +13480,7 @@ static inline int _avx512_run_container_cardinality( } CROARING_UNTARGET_AVX512 +#endif // CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY CROARING_TARGET_AVX2 CROARING_ALLOW_UNALIGNED @@ -12913,7 +13573,6 @@ static inline int _scalar_run_container_cardinality( int run_container_cardinality(const run_container_t *run) { // Empirically AVX-512 is not always faster than AVX2 -#define CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY 0 #if CROARING_COMPILER_SUPPORTS_AVX512 && \ CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY if (croaring_hardware_support() & ROARING_SUPPORTS_AVX512) { @@ -13061,11 +13720,16 @@ POSSIBILITY OF SUCH DAMAGE. #endif // _MSC_VER == 1938 #endif // __clang__ +#ifdef __FILC__ +#include +#endif + // We need portability.h to be included first, see // https://github.com/RoaringBitmap/CRoaring/issues/394 #if CROARING_REGULAR_VISUAL_STUDIO #include -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || \ + defined(__FILC__) #include #endif // CROARING_REGULAR_VISUAL_STUDIO @@ -13104,7 +13768,7 @@ unsigned int CROARING_AVX512_REQUIRED = CROARING_AVX512VBMI2 | CROARING_AVX512BITALG | CROARING_AVX512VPOPCNTDQ); #endif -#if defined(__x86_64__) || defined(_M_AMD64) // x64 +#if CROARING_IS_X64 // x64 static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) { @@ -13115,7 +13779,8 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, *ebx = cpu_info[1]; *ecx = cpu_info[2]; *edx = cpu_info[3]; -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || \ + defined(__FILC__) uint32_t level = *eax; __get_cpuid(level, eax, ebx, ecx, edx); #else @@ -13131,6 +13796,8 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, static inline uint64_t xgetbv(void) { #if defined(_MSC_VER) return _xgetbv(0); +#elif defined(__FILC__) + return zxgetbv(); #else uint32_t xcr0_lo, xcr0_hi; __asm__("xgetbv\n\t" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); @@ -13249,7 +13916,7 @@ static inline uint32_t dynamic_croaring_detect_supported_architectures(void) { #endif // end SIMD extension detection code -#if defined(__x86_64__) || defined(_M_AMD64) // x64 +#if CROARING_IS_X64 // x64 #if CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_CPP static inline uint32_t croaring_detect_supported_architectures(void) { @@ -13331,7 +13998,7 @@ int croaring_hardware_support(void) { } #endif -#endif // defined(__x86_64__) || defined(_M_AMD64) // x64 +#endif // CROARING_IS_X64 // x64 #ifdef __cplusplus } } @@ -13866,15 +14533,17 @@ size_t ra_portable_size_in_bytes(const roaring_array_t *ra) { return count; } -// This function is endian-sensitive. +// The portable serialization format is little-endian. On big-endian hosts we +// byte-swap multi-byte fields before writing them to the buffer. size_t ra_portable_serialize(const roaring_array_t *ra, char *buf) { char *initbuf = buf; uint32_t startOffset = 0; bool hasrun = ra_has_run_container(ra); if (hasrun) { uint32_t cookie = SERIAL_COOKIE | ((uint32_t)(ra->size - 1) << 16); - memcpy(buf, &cookie, sizeof(cookie)); - buf += sizeof(cookie); + uint32_t cookie_le = croaring_htole32(cookie); + memcpy(buf, &cookie_le, sizeof(cookie_le)); + buf += sizeof(cookie_le); uint32_t s = (ra->size + 7) / 8; memset(buf, 0, s); for (int32_t i = 0; i < ra->size; ++i) { @@ -13891,30 +14560,34 @@ size_t ra_portable_serialize(const roaring_array_t *ra, char *buf) { } } else { // backwards compatibility uint32_t cookie = SERIAL_COOKIE_NO_RUNCONTAINER; - - memcpy(buf, &cookie, sizeof(cookie)); - buf += sizeof(cookie); - memcpy(buf, &ra->size, sizeof(ra->size)); - buf += sizeof(ra->size); + uint32_t cookie_le = croaring_htole32(cookie); + memcpy(buf, &cookie_le, sizeof(cookie_le)); + buf += sizeof(cookie_le); + uint32_t size_le = croaring_htole32((uint32_t)ra->size); + memcpy(buf, &size_le, sizeof(size_le)); + buf += sizeof(size_le); startOffset = 4 + 4 + 4 * ra->size + 4 * ra->size; } for (int32_t k = 0; k < ra->size; ++k) { - memcpy(buf, &ra->keys[k], sizeof(ra->keys[k])); - buf += sizeof(ra->keys[k]); + uint16_t key_le = croaring_htole16(ra->keys[k]); + memcpy(buf, &key_le, sizeof(key_le)); + buf += sizeof(key_le); // get_cardinality returns a value in [1,1<<16], subtracting one // we get [0,1<<16 - 1] which fits in 16 bits uint16_t card = (uint16_t)(container_get_cardinality(ra->containers[k], ra->typecodes[k]) - 1); - memcpy(buf, &card, sizeof(card)); - buf += sizeof(card); + uint16_t card_le = croaring_htole16(card); + memcpy(buf, &card_le, sizeof(card_le)); + buf += sizeof(card_le); } if ((!hasrun) || (ra->size >= NO_OFFSET_THRESHOLD)) { // writing the containers offsets for (int32_t k = 0; k < ra->size; k++) { - memcpy(buf, &startOffset, sizeof(startOffset)); - buf += sizeof(startOffset); + uint32_t off_le = croaring_htole32(startOffset); + memcpy(buf, &off_le, sizeof(off_le)); + buf += sizeof(off_le); startOffset = startOffset + container_size_in_bytes(ra->containers[k], ra->typecodes[k]); @@ -13938,6 +14611,7 @@ size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { if (bytestotal > maxbytes) return 0; uint32_t cookie; memcpy(&cookie, buf, sizeof(int32_t)); + cookie = croaring_letoh32(cookie); buf += sizeof(uint32_t); if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) { @@ -13950,7 +14624,9 @@ size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { else { bytestotal += sizeof(int32_t); if (bytestotal > maxbytes) return 0; - memcpy(&size, buf, sizeof(int32_t)); + uint32_t size_le; + memcpy(&size_le, buf, sizeof(int32_t)); + size = (int32_t)croaring_letoh32(size_le); buf += sizeof(uint32_t); } if (size > (1 << 16) || size < 0) { @@ -13979,6 +14655,7 @@ size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { for (int32_t k = 0; k < size; ++k) { uint16_t tmp; memcpy(&tmp, keyscards + 4 * k + 2, sizeof(tmp)); + tmp = croaring_letoh16(tmp); uint32_t thiscard = tmp + 1; bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); bool isrun = false; @@ -13999,6 +14676,7 @@ size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { if (bytestotal > maxbytes) return 0; uint16_t n_runs; memcpy(&n_runs, buf, sizeof(uint16_t)); + n_runs = croaring_letoh16(n_runs); buf += sizeof(uint16_t); size_t containersize = n_runs * sizeof(rle16_t); bytestotal += containersize; @@ -14019,7 +14697,8 @@ size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { // cannot be found. If it returns true, readbytes is populated by how many bytes // were read, we have that *readbytes <= maxbytes. // -// This function is endian-sensitive. +// The portable serialization format is little-endian. On big-endian hosts we +// byte-swap multi-byte fields after reading them from the buffer. bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, const size_t maxbytes, size_t *readbytes) { *readbytes = sizeof(int32_t); // for cookie @@ -14029,6 +14708,7 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, } uint32_t cookie; memcpy(&cookie, buf, sizeof(int32_t)); + cookie = croaring_letoh32(cookie); buf += sizeof(uint32_t); if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) { @@ -14045,7 +14725,9 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, // Ran out of bytes while reading second part of the cookie. return false; } - memcpy(&size, buf, sizeof(int32_t)); + uint32_t size_le; + memcpy(&size_le, buf, sizeof(int32_t)); + size = (int32_t)croaring_letoh32(size_le); buf += sizeof(uint32_t); } if (size < 0) { @@ -14087,7 +14769,7 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, for (int32_t k = 0; k < size; ++k) { uint16_t tmp; memcpy(&tmp, keyscards + 4 * k, sizeof(tmp)); - answer->keys[k] = tmp; + answer->keys[k] = croaring_letoh16(tmp); } if ((!hasrun) || (size >= NO_OFFSET_THRESHOLD)) { *readbytes += size * 4; @@ -14105,6 +14787,7 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, for (int32_t k = 0; k < size; ++k) { uint16_t tmp; memcpy(&tmp, keyscards + 4 * k + 2, sizeof(tmp)); + tmp = croaring_letoh16(tmp); uint32_t thiscard = tmp + 1; bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); bool isrun = false; @@ -14126,7 +14809,7 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, return false; } // it is now safe to read - bitset_container_t *c = bitset_container_create(); + bitset_container_t *c = bitset_container_create_uninitialized(); if (c == NULL) { // memory allocation failure // Failed to allocate memory for a bitset container. ra_clear(answer); // we need to clear the containers already @@ -14148,6 +14831,7 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, } uint16_t n_runs; memcpy(&n_runs, buf, sizeof(uint16_t)); + n_runs = croaring_letoh16(n_runs); size_t containersize = n_runs * sizeof(rle16_t); *readbytes += containersize; if (*readbytes > maxbytes) { // data is corrupted? @@ -15406,6 +16090,113 @@ roaring_bitmap_t *roaring_bitmap_or(const roaring_bitmap_t *x1, return answer; } +static void roaring_inplace_merge_bulk(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2, int dst, + int left, int right, bool is_xor) { + roaring_array_t *ra1 = &x1->high_low_container; + const roaring_array_t *ra2 = &x2->high_low_container; + const bool cow2 = is_cow(x2); + const int length1 = ra1->size; + const int length2 = ra2->size; + + int distinct = 0; + { + int l = left, r = right; + while (l < length1 && r < length2) { + uint16_t k1 = ra1->keys[l]; + uint16_t k2 = ra2->keys[r]; + if (k1 < k2) { + l++; + } else if (k1 > k2) { + r++; + } else { + l++; + r++; + } + distinct++; + } + distinct += (length1 - l) + (length2 - r); + } + const int total = dst + distinct; + + roaring_array_t merged; + ra_init_with_capacity(&merged, total > 0 ? (uint32_t)total : 1); + + for (int i = 0; i < dst; i++) { + ra_append(&merged, ra1->keys[i], ra1->containers[i], ra1->typecodes[i]); + } + + uint8_t result_type = 0; + while (left < length1 && right < length2) { + uint16_t k1 = ra1->keys[left]; + uint16_t k2 = ra2->keys[right]; + if (k1 < k2) { + ra_append(&merged, k1, ra1->containers[left], ra1->typecodes[left]); + left++; + } else if (k1 > k2) { + uint8_t type2 = ra2->typecodes[right]; + container_t *c2 = + get_copy_of_container(ra2->containers[right], &type2, cow2); + if (cow2) { + ra_set_container_at_index(ra2, right, c2, type2); + } + ra_append(&merged, k2, c2, type2); + right++; + } else { + uint8_t type1 = ra1->typecodes[left]; + container_t *c1 = ra1->containers[left]; + uint8_t type2 = ra2->typecodes[right]; + container_t *c2 = ra2->containers[right]; + if (is_xor) { + container_t *c; + if (type1 == SHARED_CONTAINER_TYPE) { + c = container_xor(c1, type1, c2, type2, &result_type); + shared_container_free(CAST_shared(c1)); + } else { + c = container_ixor(c1, type1, c2, type2, &result_type); + } + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&merged, k1, c, result_type); + } else { + container_free(c, result_type); + } + } else { + if (container_is_full(c1, type1)) { + ra_append(&merged, k1, c1, type1); + } else { + container_t *c = + (type1 == SHARED_CONTAINER_TYPE) + ? container_or(c1, type1, c2, type2, &result_type) + : container_ior(c1, type1, c2, type2, &result_type); + if (c != c1) { + container_free(c1, type1); + } + ra_append(&merged, k1, c, result_type); + } + } + left++; + right++; + } + } + for (; left < length1; left++) { + ra_append(&merged, ra1->keys[left], ra1->containers[left], + ra1->typecodes[left]); + } + for (; right < length2; right++) { + uint8_t type2 = ra2->typecodes[right]; + container_t *c2 = + get_copy_of_container(ra2->containers[right], &type2, cow2); + if (cow2) { + ra_set_container_at_index(ra2, right, c2, type2); + } + ra_append(&merged, ra2->keys[right], c2, type2); + } + + merged.flags = ra1->flags; + ra_clear_without_containers(ra1); + *ra1 = merged; +} + // inplace or (modifies its first argument). void roaring_bitmap_or_inplace(roaring_bitmap_t *x1, const roaring_bitmap_t *x2) { @@ -15455,22 +16246,8 @@ void roaring_bitmap_or_inplace(roaring_bitmap_t *x1, s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - - // container_t *c2_clone = container_clone(c2, type2); - ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, - type2); - pos1++; - length1++; - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + roaring_inplace_merge_bulk(x1, x2, pos1, pos1, pos2, false); + return; } } if (pos1 == length1) { @@ -15606,8 +16383,9 @@ void roaring_bitmap_xor_inplace(roaring_bitmap_t *x1, ++pos1; } else { container_free(c, result_type); - ra_remove_at_index(&x1->high_low_container, pos1); - --length1; + roaring_inplace_merge_bulk(x1, x2, pos1, pos1 + 1, pos2 + 1, + true); + return; } ++pos2; @@ -15622,21 +16400,8 @@ void roaring_bitmap_xor_inplace(roaring_bitmap_t *x1, s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - - ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, - type2); - pos1++; - length1++; - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + roaring_inplace_merge_bulk(x1, x2, pos1, pos1, pos2, true); + return; } } if (pos1 == length1) { @@ -15973,9 +16738,18 @@ size_t roaring_bitmap_serialize(const roaring_bitmap_t *r, char *buf) { return roaring_bitmap_portable_serialize(r, buf + 1) + 1; } else { buf[0] = CROARING_SERIALIZATION_ARRAY_UINT32; - memcpy(buf + 1, &cardinality, sizeof(uint32_t)); - roaring_bitmap_to_uint32_array( - r, (uint32_t *)(buf + 1 + sizeof(uint32_t))); + uint32_t card_le = croaring_htole32((uint32_t)cardinality); + memcpy(buf + 1, &card_le, sizeof(uint32_t)); + uint32_t *out = (uint32_t *)(buf + 1 + sizeof(uint32_t)); + roaring_bitmap_to_uint32_array(r, out); +#if CROARING_IS_BIG_ENDIAN + for (uint64_t i = 0; i < cardinality; ++i) { + uint32_t v; + memcpy(&v, out + i, sizeof(uint32_t)); + v = croaring_htole32(v); + memcpy(out + i, &v, sizeof(uint32_t)); + } +#endif return 1 + (size_t)sizeasarray; } } @@ -16034,6 +16808,7 @@ roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf) { uint32_t card; memcpy(&card, bufaschar + 1, sizeof(uint32_t)); + card = croaring_letoh32(card); const uint32_t *elems = (const uint32_t *)(bufaschar + 1 + sizeof(uint32_t)); @@ -16047,6 +16822,7 @@ roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf) { // elems may not be aligned, read with memcpy uint32_t elem; memcpy(&elem, elems + i, sizeof(elem)); + elem = croaring_letoh32(elem); roaring_bitmap_add_bulk(bitmap, &context, elem); } return bitmap; @@ -16072,6 +16848,7 @@ roaring_bitmap_t *roaring_bitmap_deserialize_safe(const void *buf, /* This looks like a compressed set of uint32_t elements */ uint32_t card; memcpy(&card, bufaschar + 1, sizeof(uint32_t)); + card = croaring_letoh32(card); // Check the buffer is big enough to contain card uint32_t elements if (maxbytes < 1 + sizeof(uint32_t) + card * sizeof(uint32_t)) { @@ -16090,6 +16867,7 @@ roaring_bitmap_t *roaring_bitmap_deserialize_safe(const void *buf, // elems may not be aligned, read with memcpy uint32_t elem; memcpy((char *)&elem, (char *)(elems + i), sizeof(elem)); + elem = croaring_letoh32(elem); roaring_bitmap_add_bulk(bitmap, &context, elem); } return bitmap; @@ -16326,7 +17104,7 @@ uint32_t roaring_uint32_iterator_read(roaring_uint32_iterator_t *it, it->has_value = true; it->current_value = it->highbits | low16; // If the container still has values, we must have stopped because - // we skipped enough values. + // we read enough values. assert(ret == count); return ret; } @@ -16336,6 +17114,31 @@ uint32_t roaring_uint32_iterator_read(roaring_uint32_iterator_t *it, return ret; } +uint32_t roaring_uint32_iterator_read_backward(roaring_uint32_iterator_t *it, + uint32_t *buf, uint32_t count) { + uint32_t ret = 0; + while (it->has_value && ret < count) { + uint32_t consumed; + uint16_t low16 = (uint16_t)it->current_value; + bool has_value = container_iterator_read_backward_into_uint32( + it->container, it->typecode, &it->container_it, it->highbits, buf, + count - ret, &consumed, &low16); + ret += consumed; + buf += consumed; + if (has_value) { + it->has_value = true; + it->current_value = it->highbits | low16; + // If the container still has values, we must have stopped because + // we read enough values. + assert(ret == count); + return ret; + } + it->container_index--; + it->has_value = loadlastvalue(it); + } + return ret; +} + uint32_t roaring_uint32_iterator_skip(roaring_uint32_iterator_t *it, uint32_t count) { uint32_t ret = 0; @@ -16386,6 +17189,72 @@ uint32_t roaring_uint32_iterator_skip_backward(roaring_uint32_iterator_t *it, return ret; } +size_t roaring_uint32_iterator_read_ranges(roaring_uint32_iterator_t *it, + roaring_uint32_range_closed_t *buf, + size_t count) { + size_t ret = 0; + while (it->has_value && ret < count) { + buf[ret].min = it->current_value; + for (;;) { + uint16_t low16 = (uint16_t)it->current_value; + bool container_has_more; + uint16_t run_end_low16 = container_iterator_find_run_end( + it->container, it->typecode, &it->container_it, &low16, + &container_has_more); + buf[ret].max = it->highbits | run_end_low16; + + if (container_has_more) { + it->current_value = it->highbits | low16; + break; + } + // Move to next container + it->container_index++; + it->has_value = loadfirstvalue(it); + // Continue merging only if the run reached the container + // boundary and the next container starts exactly at max+1. + if (run_end_low16 != UINT16_MAX || !it->has_value || + it->current_value != buf[ret].max + 1) { + break; + } + } + ret++; + } + return ret; +} + +size_t roaring_uint32_iterator_read_prev_ranges( + roaring_uint32_iterator_t *it, roaring_uint32_range_closed_t *buf, + size_t count) { + size_t ret = 0; + while (it->has_value && ret < count) { + buf[ret].max = it->current_value; + for (;;) { + uint16_t low16 = (uint16_t)it->current_value; + bool container_has_more; + uint16_t run_start_low16 = container_iterator_find_run_start( + it->container, it->typecode, &it->container_it, &low16, + &container_has_more); + buf[ret].min = it->highbits | run_start_low16; + + if (container_has_more) { + it->current_value = it->highbits | low16; + break; + } + // Move to previous container + it->container_index--; + it->has_value = loadlastvalue(it); + // Continue merging only if the run reached the container + // boundary and the previous container ends exactly at min-1. + if (run_start_low16 != 0 || !it->has_value || + it->current_value != buf[ret].min - 1) { + break; + } + } + ret++; + } + return ret; +} + void roaring_uint32_iterator_free(roaring_uint32_iterator_t *it) { roaring_free(it); } @@ -17173,6 +18042,10 @@ void roaring_bitmap_rank_many(const roaring_bitmap_t *bm, const uint32_t *begin, iter++; } } + while (iter != end) { // must have N outputs for N inputs... + *(ans++) = size; // ...everything left is beyond all containers + iter++; + } } /** @@ -17722,6 +18595,15 @@ const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, CROARING_ALLOW_UNALIGNED roaring_bitmap_t *roaring_bitmap_portable_deserialize_frozen(const char *buf) { +#if CROARING_IS_BIG_ENDIAN + // The portable format is little-endian on every host, and this function + // uses the container payloads where they sit rather than converting them. + // There is therefore no correct in-place view of them here: refuse rather + // than hand back a bitmap that silently reads byte-swapped values. Use + // roaring_bitmap_portable_deserialize_safe(), which converts as it copies. + (void)buf; + return NULL; +#else char *start_of_buf = (char *)buf; uint32_t cookie; int32_t num_containers; @@ -17878,6 +18760,7 @@ roaring_bitmap_t *roaring_bitmap_portable_deserialize_frozen(const char *buf) { } return rb; +#endif } bool roaring_bitmap_to_bitset(const roaring_bitmap_t *r, bitset_t *bitset) { @@ -17930,8 +18813,8 @@ bool roaring_bitmap_to_bitset(const roaring_bitmap_t *r, bitset_t *bitset) { /* end file src/roaring.c */ /* begin file src/roaring64.c */ #include -#include #include +#include #include #include @@ -17962,6 +18845,9 @@ typedef struct roaring64_bitmap_s { uint64_t first_free; uint64_t capacity; container_t **containers; + // Parallel to containers[]. Live slots (non-NULL pointers) have the + // matching typecode; NULL slots are skipped and their typecodes ignored. + uint8_t *typecodes; } roaring64_bitmap_t; // Leaf type of the ART used to keep the high 48 bits of each entry. @@ -17971,25 +18857,76 @@ typedef roaring64_leaf_t leaf_t; // Iterator struct to hold iteration state. typedef struct roaring64_iterator_s { - const roaring64_bitmap_t *r; - art_iterator_t art_it; - roaring_container_iterator_t container_it; + // The order here is deliberate: everything `roaring64_iterator_advance` + // touches per value is packed into the first 64 bytes, and `art_it` -- + // 136 bytes, of which only `art_it.value` is read per value -- is last. + // Putting `art_it` earlier pushes the rest past the first cache line and + // costs ~18% on a scalar iteration loop. + + // Must stay first, and must stay a `roaring64_iterator_public_t`: the + // inline `roaring64_iterator_value` / `roaring64_iterator_has_value` in + // the public header reach these two members by converting a + // `roaring64_iterator_t *` to a pointer to its initial member. + roaring64_iterator_public_t pub; + uint64_t high48; // Key that art_it points to. + roaring_container_iterator_t container_it; - uint64_t value; - bool has_value; + // Forward-iteration cache for bitset containers. `container_iterator_next` + // recomputes the word index from `container_it.index`, reloads the word + // and re-masks it for every value; keeping the remaining bits of the + // current word here turns that into a `tzcnt` and a `blsr`. Array and run + // containers already advance by a single increment and gain nothing from + // a cache, so they stay on the ordinary path. + // + // Every field is a pure function of the ART position and + // `container_it.index`, so the cache describes where the iterator is + // exactly when both still match what it was built from. `fast_type` is + // BITSET_CONTAINER_TYPE, or 0 when there is no usable cache. + uint32_t fast_wordindex; + const art_val_t *fast_art_value; + const uint64_t *fast_words; + uint64_t fast_word; + int32_t fast_index; + uint8_t fast_type; // If has_value is false, then the iterator is saturated. This field // indicates the direction of saturation. If true, there are no more values // in the forward direction. If false, there are no more values in the // backward direction. bool saturated_forward; + + const roaring64_bitmap_t *r; + art_iterator_t art_it; } roaring64_iterator_t; static inline bool is_frozen64(const roaring64_bitmap_t *r) { return r->flags & ROARING_FLAG_FROZEN; } +static inline bool is_frozen_art64(const roaring64_bitmap_t *r) { + return r->flags & ROARING_FLAG_FROZEN_ART; +} + +typedef union { + bitset_container_t bitset; + array_container_t array; + run_container_t run; +} frozen_container_header_t; + +static void *roaring64_arena_alloc(char **arena, size_t num_bytes) { + char *res = *arena; + *arena += num_bytes; + return res; +} + +static char *roaring64_arena_pad(char *cursor, const char *base, + size_t alignment) { + uint64_t off = (uint64_t)(cursor - base); + uint64_t aligned = (off + alignment - 1) & ~(uint64_t)(alignment - 1); + return (char *)base + aligned; +} + // Splits the given uint64 key into high 48 bit and low 16 bit components. // Expects high48_out to be of length ART_KEY_BYTES. static inline uint16_t split_key(uint64_t key, uint8_t high48_out[]) { @@ -18023,19 +18960,26 @@ static inline container_t *get_container(const roaring64_bitmap_t *r, return r->containers[get_index(leaf)]; } +// Writes the pointer and its typecode together so they cannot drift. +static inline void set_container_at(roaring64_bitmap_t *r, uint64_t index, + container_t *container, uint8_t typecode) { + r->containers[index] = container; + r->typecodes[index] = typecode; +} + // Replaces the container of `leaf` with the given container. Returns the // modified leaf for convenience. static inline leaf_t replace_container(roaring64_bitmap_t *r, leaf_t *leaf, container_t *container, uint8_t typecode) { uint64_t index = get_index(*leaf); - r->containers[index] = container; + set_container_at(r, index, container, typecode); *leaf = create_leaf(index, typecode); return *leaf; } /** - * Extends the array of container pointers. + * Extends the array of container pointers (and the parallel typecode array). */ static void extend_containers(roaring64_bitmap_t *r) { uint64_t size = r->first_free; @@ -18054,6 +18998,8 @@ static void extend_containers(roaring64_bitmap_t *r) { r->containers = (container_t **)roaring_realloc( r->containers, new_capacity * sizeof(container_t *)); memset(r->containers + r->capacity, 0, increase * sizeof(container_t *)); + r->typecodes = (uint8_t *)roaring_realloc(r->typecodes, + new_capacity * sizeof(uint8_t)); r->capacity = new_capacity; } @@ -18078,10 +19024,41 @@ static uint64_t allocate_index(roaring64_bitmap_t *r) { static leaf_t add_container(roaring64_bitmap_t *r, container_t *container, uint8_t typecode) { uint64_t index = allocate_index(r); - r->containers[index] = container; + set_container_at(r, index, container, typecode); return create_leaf(index, typecode); } +static void ensure_container_capacity(roaring64_bitmap_t *r, uint64_t extra) { + uint64_t needed = r->first_free + extra; + if (needed <= r->capacity) { + return; + } + uint64_t new_capacity = r->capacity; + if (new_capacity == 0) { + new_capacity = 2; + } + while (new_capacity < needed) { + uint64_t grown; + if (new_capacity < 1024) { + grown = 2 * new_capacity; + } else { + grown = new_capacity + new_capacity / 4; + } + if (grown <= new_capacity) { + new_capacity = needed; + break; + } + new_capacity = grown; + } + uint64_t increase = new_capacity - r->capacity; + r->containers = (container_t **)roaring_realloc( + r->containers, new_capacity * sizeof(container_t *)); + memset(r->containers + r->capacity, 0, increase * sizeof(container_t *)); + r->typecodes = (uint8_t *)roaring_realloc(r->typecodes, + new_capacity * sizeof(uint8_t)); + r->capacity = new_capacity; +} + static void remove_container(roaring64_bitmap_t *r, leaf_t leaf) { uint64_t index = get_index(leaf); r->containers[index] = NULL; @@ -18105,6 +19082,36 @@ static inline int compare_high48(art_key_chunk_t key1[], return art_compare_keys(key1, key2); } +// Cache the current container so advance() need not re-read the ART leaf, +// reload the container pointer, or (for bitsets) re-mask the current word. +// Called after any positioning that sets container_it.index. +// fast_type is 0 when there is no usable cache (run containers, exhausted). +static inline void roaring64_iterator_prime(roaring64_iterator_t *it) { + it->fast_type = 0; + if (it->art_it.value == NULL) { + return; + } + leaf_t leaf = (leaf_t)*it->art_it.value; + if (get_typecode(leaf) != BITSET_CONTAINER_TYPE) { + return; + } + const bitset_container_t *bc = + const_CAST_bitset(get_container(it->r, leaf)); + int32_t index = it->container_it.index; + uint32_t wordindex = (uint32_t)index >> 6; + uint32_t bit = (uint32_t)index & 63u; + uint64_t word = bc->words[wordindex]; + // Bits strictly after the current value in this word. A shift of 64 is + // undefined, so the last bit of a word is a special case. + it->fast_word = + (bit == 63u) ? UINT64_C(0) : (word & (UINT64_MAX << (bit + 1))); + it->fast_words = bc->words; + it->fast_wordindex = wordindex; + it->fast_index = index; + it->fast_art_value = it->art_it.value; + it->fast_type = BITSET_CONTAINER_TYPE; +} + static inline bool roaring64_iterator_init_at_leaf_first( roaring64_iterator_t *it) { it->high48 = combine_key(it->art_it.key, 0); @@ -18112,8 +19119,10 @@ static inline bool roaring64_iterator_init_at_leaf_first( uint16_t low16 = 0; it->container_it = container_init_iterator(get_container(it->r, leaf), get_typecode(leaf), &low16); - it->value = it->high48 | low16; - return (it->has_value = true); + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + roaring64_iterator_prime(it); + return true; } static inline bool roaring64_iterator_init_at_leaf_last( @@ -18123,16 +19132,18 @@ static inline bool roaring64_iterator_init_at_leaf_last( uint16_t low16 = 0; it->container_it = container_init_iterator_last(get_container(it->r, leaf), get_typecode(leaf), &low16); - it->value = it->high48 | low16; - return (it->has_value = true); + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + roaring64_iterator_prime(it); + return true; } static inline roaring64_iterator_t *roaring64_iterator_init_at( const roaring64_bitmap_t *r, roaring64_iterator_t *it, bool first) { it->r = r; it->art_it = art_init_iterator((art_t *)&r->art, first); - it->has_value = it->art_it.value != NULL; - if (it->has_value) { + it->pub.has_value = it->art_it.value != NULL; + if (it->pub.has_value) { if (first) { roaring64_iterator_init_at_leaf_first(it); } else { @@ -18140,6 +19151,7 @@ static inline roaring64_iterator_t *roaring64_iterator_init_at( } } else { it->saturated_forward = first; + it->fast_type = 0; } return it; } @@ -18152,6 +19164,7 @@ roaring64_bitmap_t *roaring64_bitmap_create(void) { r->capacity = 0; r->first_free = 0; r->containers = NULL; + r->typecodes = NULL; return r; } @@ -18159,22 +19172,24 @@ void roaring64_bitmap_free(roaring64_bitmap_t *r) { if (!r) { return; } + if (is_frozen64(r)) { + // Headers, containers[], and typecodes[] live in the same allocation + // as `r`. Payloads alias a caller buffer. + if (!is_frozen_art64(r)) { + art_free(&r->art); + } + roaring_free(r); + return; + } art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); while (it.value != NULL) { leaf_t leaf = (leaf_t)*it.value; - if (is_frozen64(r)) { - // Only free the container itself, not the buffer-backed contents - // within. - roaring_free(get_container(r, leaf)); - } else { - container_free(get_container(r, leaf), get_typecode(leaf)); - } + container_free(get_container(r, leaf), get_typecode(leaf)); art_iterator_next(&it); } - if (!is_frozen64(r)) { - art_free(&r->art); - } + art_free(&r->art); roaring_free(r->containers); + roaring_free(r->typecodes); roaring_free(r); } @@ -18195,18 +19210,56 @@ roaring64_bitmap_t *roaring64_bitmap_copy(const roaring64_bitmap_t *r) { return result; } -/** - * Steal the containers from a 32-bit bitmap and insert them into a 64-bit - * bitmap (with an offset) - * - * After calling this function, the original bitmap will be empty, and the - * returned bitmap will contain all the values from the original bitmap. - */ -static void move_from_roaring32_offset(roaring64_bitmap_t *dst, +void roaring64_bitmap_overwrite(roaring64_bitmap_t *dest, + const roaring64_bitmap_t *src) { + if (dest == src) { + return; + } + + // Free dest's containers. + art_iterator_t it = art_init_iterator(&dest->art, /*first=*/true); + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + container_free(get_container(dest, leaf), get_typecode(leaf)); + art_iterator_next(&it); + } + art_free(&dest->art); + + // Reinitialize dest. + art_init_cleared(&dest->art); + dest->flags = 0; + dest->first_free = 0; + if (dest->capacity > 0) { + memset(dest->containers, 0, + sizeof(dest->containers[0]) * dest->capacity); + } + + // Copy src's containers into dest. + it = art_init_iterator((art_t *)&src->art, /*first=*/true); + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + uint8_t typecode = get_typecode(leaf); + container_t *container = get_copy_of_container( + get_container(src, leaf), &typecode, /*copy_on_write=*/false); + leaf_t dest_leaf = add_container(dest, container, typecode); + art_insert(&dest->art, it.key, (art_val_t)dest_leaf); + art_iterator_next(&it); + } +} + +/** + * Steal the containers from a 32-bit bitmap and insert them into a 64-bit + * bitmap (with an offset) + * + * After calling this function, the original bitmap will be empty, and the + * returned bitmap will contain all the values from the original bitmap. + */ +static void move_from_roaring32_offset(roaring64_bitmap_t *dst, roaring_bitmap_t *src, uint32_t high_bits) { uint64_t key_base = ((uint64_t)high_bits) << 32; uint32_t r32_size = ra_get_size(&src->high_low_container); + ensure_container_capacity(dst, r32_size); for (uint32_t i = 0; i < r32_size; ++i) { uint16_t key = ra_get_key_at_index(&src->high_low_container, i); uint8_t typecode; @@ -18447,12 +19500,20 @@ bool roaring64_bitmap_contains_range(const roaring64_bitmap_t *r, uint64_t min, if (min >= max) { return true; } + return roaring64_bitmap_contains_range_closed(r, min, max - 1); +} + +bool roaring64_bitmap_contains_range_closed(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max) { + if (min > max) { + return true; + } uint8_t min_high48[ART_KEY_BYTES]; uint16_t min_low16 = split_key(min, min_high48); uint8_t max_high48[ART_KEY_BYTES]; uint16_t max_low16 = split_key(max, max_high48); - uint64_t max_high48_bits = (max - 1) & 0xFFFFFFFFFFFF0000; // Inclusive + uint64_t max_high48_bits = max & 0xFFFFFFFFFFFF0000; art_iterator_t it = art_lower_bound((art_t *)&r->art, min_high48); if (it.value == NULL || combine_key(it.key, 0) > min) { @@ -18478,7 +19539,7 @@ bool roaring64_bitmap_contains_range(const roaring64_bitmap_t *r, uint64_t min, } uint32_t container_max = 0xFFFF + 1; // Exclusive if (compare_high48(it.key, max_high48) == 0) { - container_max = max_low16; + container_max = (uint32_t)max_low16 + 1; } // For the first and last containers we use container_contains_range, @@ -18674,7 +19735,7 @@ void roaring64_bitmap_remove_bulk(roaring64_bitmap_t *r, } if (!container_nonzero_cardinality(container2, typecode2)) { container_free(container2, typecode2); - leaf_t leaf; + leaf_t leaf = 0; bool erased = art_erase(art, high48, (art_val_t *)&leaf); assert(erased); (void)erased; @@ -18760,7 +19821,7 @@ void roaring64_bitmap_remove_range_closed(roaring64_bitmap_t *r, uint64_t min, art_iterator_t it = art_upper_bound(art, min_high48); while (it.value != NULL && art_compare_keys(it.key, max_high48) < 0) { - leaf_t leaf; + leaf_t leaf = 0; bool erased = art_iterator_erase(&it, (art_val_t *)&leaf); assert(erased); (void)erased; @@ -18775,13 +19836,15 @@ void roaring64_bitmap_clear(roaring64_bitmap_t *r) { } uint64_t roaring64_bitmap_get_cardinality(const roaring64_bitmap_t *r) { - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + // Scan the pointer array rather than the ART: the arrays are sequential + // and the ART is not. first_free is a free-list head, not a size, so + // after deletes live containers can sit above it; skip NULL slots. uint64_t cardinality = 0; - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - cardinality += container_get_cardinality(get_container(r, leaf), - get_typecode(leaf)); - art_iterator_next(&it); + for (uint64_t i = 0; i < r->capacity; ++i) { + if (r->containers[i] != NULL) { + cardinality += + container_get_cardinality(r->containers[i], r->typecodes[i]); + } } return cardinality; } @@ -18863,6 +19926,26 @@ uint64_t roaring64_bitmap_maximum(const roaring64_bitmap_t *r) { it.key, container_maximum(get_container(r, leaf), get_typecode(leaf))); } +bool roaring64_bitmap_remove_run_compression(roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + bool removed = false; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + if (get_typecode(*leaf) == RUN_CONTAINER_TYPE) { + run_container_t *run = CAST_run(get_container(r, *leaf)); + int32_t card = run_container_cardinality(run); + uint8_t new_typecode; + container_t *new_container = + convert_to_bitset_or_array_container(run, card, &new_typecode); + run_container_free(run); + replace_container(r, leaf, new_container, new_typecode); + removed = true; + } + art_iterator_next(&it); + } + return removed; +} + bool roaring64_bitmap_run_optimize(roaring64_bitmap_t *r) { art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); bool has_run_container = false; @@ -18885,9 +19968,10 @@ static void move_to_shrink(roaring64_bitmap_t *r, leaf_t *leaf) { if (idx < r->first_free) { return; } - r->containers[r->first_free] = get_container(r, *leaf); + uint8_t typecode = get_typecode(*leaf); + set_container_at(r, r->first_free, get_container(r, *leaf), typecode); r->containers[idx] = NULL; - *leaf = create_leaf(r->first_free, get_typecode(*leaf)); + *leaf = create_leaf(r->first_free, typecode); r->first_free = next_free_container_idx(r); } @@ -18912,7 +19996,10 @@ size_t roaring64_bitmap_shrink_to_fit(roaring64_bitmap_t *r) { if (new_capacity < r->capacity) { r->containers = (container_t **)roaring_realloc( r->containers, new_capacity * sizeof(container_t *)); - freed += (r->capacity - new_capacity) * sizeof(container_t *); + r->typecodes = (uint8_t *)roaring_realloc( + r->typecodes, new_capacity * sizeof(uint8_t)); + freed += (r->capacity - new_capacity) * + (sizeof(container_t *) + sizeof(uint8_t)); r->capacity = new_capacity; } return freed; @@ -18968,8 +20055,17 @@ static bool roaring64_leaf_internal_validate(const art_val_t val, void *context) { leaf_t leaf = (leaf_t)val; roaring64_bitmap_t *r = (roaring64_bitmap_t *)context; - return container_internal_validate(get_container(r, leaf), - get_typecode(leaf), reason); + uint64_t index = get_index(leaf); + uint8_t typecode = get_typecode(leaf); + if (index >= r->capacity || r->containers[index] == NULL) { + *reason = "ART leaf points at an empty container slot"; + return false; + } + if (r->typecodes[index] != typecode) { + *reason = "typecode array does not match ART leaf"; + return false; + } + return container_internal_validate(r->containers[index], typecode, reason); } bool roaring64_bitmap_internal_validate(const roaring64_bitmap_t *r, @@ -19180,7 +20276,7 @@ void roaring64_bitmap_and_inplace(roaring64_bitmap_t *r1, if (!it2_present || compare_result < 0) { // Cases 1 and 3a: it1 is the only iterator or is before it2. - leaf_t leaf; + leaf_t leaf = 0; bool erased = art_iterator_erase(&it1, (art_val_t *)&leaf); assert(erased); (void)erased; @@ -19821,6 +20917,131 @@ void roaring64_bitmap_flip_closed_inplace(roaring64_bitmap_t *r, uint64_t min, } } +roaring64_bitmap_t *roaring64_bitmap_add_offset_signed( + const roaring64_bitmap_t *r, bool positive, uint64_t offset) { + if (offset == 0) { + return roaring64_bitmap_copy(r); + } + + roaring64_bitmap_t *answer = roaring64_bitmap_create(); + + // Decompose the offset into a signed container-level shift and an + // intra-container shift. For negative offsets the low 16 bits wrap: e.g. + // -1 = container_offset(-1) + in_offset(0xffff), because shifting by -1 + // container is a shift of -0x1_0000, so we need to shift up within + // containers to get back to -1 + uint16_t low16 = (uint16_t)offset; + int64_t container_offset; + uint16_t in_offset; + if (positive) { + container_offset = (int64_t)(offset >> 16); + in_offset = low16; + } else if (low16 == 0) { + container_offset = -(int64_t)(offset >> 16); + in_offset = 0; + } else { + container_offset = -(int64_t)(offset >> 16) - 1; + in_offset = (uint16_t)-low16; + } + + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + + if (in_offset == 0) { + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + int64_t k = + (int64_t)(combine_key(it.key, 0) >> 16) + container_offset; + if ((uint64_t)k < (uint64_t)1 << 48) { + uint8_t new_high48[ART_KEY_BYTES]; + split_key((uint64_t)k << 16, new_high48); + uint8_t typecode = get_typecode(leaf); + container_t *container = + get_copy_of_container(get_container(r, leaf), &typecode, + /*copy_on_write=*/false); + leaf_t new_leaf = add_container(answer, container, typecode); + art_insert(&answer->art, new_high48, (art_val_t)new_leaf); + } + art_iterator_next(&it); + } + return answer; + } + + // Track the most recently inserted hi container so that the next + // iteration's lo can merge with it without re-searching the ART. + leaf_t *prev_hi_leaf = NULL; + int64_t prev_hi_k = -1; + + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + int64_t k = (int64_t)(combine_key(it.key, 0) >> 16) + container_offset; + + container_t *lo = NULL, *hi = NULL; + container_t **lo_ptr = NULL, **hi_ptr = NULL; + + if ((uint64_t)k < (uint64_t)1 << 48) { + lo_ptr = &lo; + } + if ((uint64_t)(k + 1) < (uint64_t)1 << 48) { + hi_ptr = &hi; + } + if (lo_ptr == NULL && hi_ptr == NULL) { + art_iterator_next(&it); + continue; + } + + uint8_t typecode = get_typecode(leaf); + const container_t *c = + container_unwrap_shared(get_container(r, leaf), &typecode); + container_add_offset(c, typecode, lo_ptr, hi_ptr, in_offset); + + if (lo != NULL) { + if (prev_hi_leaf != NULL && prev_hi_k == k) { + uint8_t existing_type = get_typecode(*prev_hi_leaf); + container_t *existing_c = get_container(answer, *prev_hi_leaf); + uint8_t merged_type; + container_t *merged_c = container_ior( + existing_c, existing_type, lo, typecode, &merged_type); + if (merged_c != existing_c) { + container_free(existing_c, existing_type); + } + replace_container(answer, prev_hi_leaf, merged_c, merged_type); + container_free(lo, typecode); + } else { + uint8_t lo_high48[ART_KEY_BYTES]; + split_key((uint64_t)k << 16, lo_high48); + leaf_t new_leaf = add_container(answer, lo, typecode); + art_insert(&answer->art, lo_high48, (art_val_t)new_leaf); + } + } + + prev_hi_leaf = NULL; + if (hi != NULL) { + uint8_t hi_high48[ART_KEY_BYTES]; + split_key((uint64_t)(k + 1) << 16, hi_high48); + leaf_t new_leaf = add_container(answer, hi, typecode); + prev_hi_leaf = (leaf_t *)art_insert(&answer->art, hi_high48, + (art_val_t)new_leaf); + prev_hi_k = k + 1; + } + + art_iterator_next(&it); + } + + // Repair containers (e.g., convert low-cardinality bitset containers to + // array containers after lazy union operations). + art_iterator_t repair_it = art_init_iterator(&answer->art, /*first=*/true); + while (repair_it.value != NULL) { + leaf_t *leaf_ptr = (leaf_t *)repair_it.value; + uint8_t typecode = get_typecode(*leaf_ptr); + container_t *repaired = container_repair_after_lazy( + get_container(answer, *leaf_ptr), &typecode); + replace_container(answer, leaf_ptr, repaired, typecode); + art_iterator_next(&repair_it); + } + + return answer; +} + // Returns the number of distinct high 32-bit entries in the bitmap. static inline uint64_t count_high32(const roaring64_bitmap_t *r) { art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); @@ -19915,7 +21136,8 @@ size_t roaring64_bitmap_portable_serialize(const roaring64_bitmap_t *r, // Write as uint64 the distinct number of "buckets", where a bucket is // defined as the most significant 32 bits of an element. uint64_t high32_count = count_high32(r); - memcpy(buf, &high32_count, sizeof(high32_count)); + uint64_t high32_count_le = croaring_htole64(high32_count); + memcpy(buf, &high32_count_le, sizeof(high32_count_le)); buf += sizeof(high32_count); art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); @@ -19930,7 +21152,8 @@ size_t roaring64_bitmap_portable_serialize(const roaring64_bitmap_t *r, if (bitmap32 != NULL) { // Write as uint32 the most significant 32 bits of the // bucket. - memcpy(buf, &prev_high32, sizeof(prev_high32)); + uint32_t prev_high32_le = croaring_htole32(prev_high32); + memcpy(buf, &prev_high32_le, sizeof(prev_high32_le)); buf += sizeof(prev_high32); // Write the 32-bit Roaring bitmaps representing the least @@ -19961,7 +21184,8 @@ size_t roaring64_bitmap_portable_serialize(const roaring64_bitmap_t *r, if (bitmap32 != NULL) { // Write as uint32 the most significant 32 bits of the bucket. - memcpy(buf, &prev_high32, sizeof(prev_high32)); + uint32_t prev_high32_le = croaring_htole32(prev_high32); + memcpy(buf, &prev_high32_le, sizeof(prev_high32_le)); buf += sizeof(prev_high32); // Write the 32-bit Roaring bitmaps representing the least @@ -19988,6 +21212,7 @@ size_t roaring64_bitmap_portable_deserialize_size(const char *buf, return 0; } memcpy(&buckets, buf, sizeof(buckets)); + buckets = croaring_letoh64(buckets); buf += sizeof(buckets); read_bytes += sizeof(buckets); @@ -20034,6 +21259,7 @@ roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe( return NULL; } memcpy(&buckets, buf, sizeof(buckets)); + buckets = croaring_letoh64(buckets); buf += sizeof(buckets); read_bytes += sizeof(buckets); @@ -20053,6 +21279,7 @@ roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe( return NULL; } memcpy(&high32, buf, sizeof(high32)); + high32 = croaring_letoh32(high32); buf += sizeof(high32); read_bytes += sizeof(high32); // High 32 bits must be strictly increasing. @@ -20063,22 +21290,26 @@ roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe( previous_high32 = high32; // Read the 32-bit Roaring bitmaps representing the least - // significant bits of a set of elements. - size_t bitmap32_size = roaring_bitmap_portable_deserialize_size( - buf, maxbytes - read_bytes); - if (bitmap32_size == 0) { + // significant bits of a set of elements. ra_portable_deserialize + // already reports bytes consumed, so we do not walk the 32-bit + // format a second time with deserialize_size. + roaring_bitmap_t *bitmap32 = + (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); + if (bitmap32 == NULL) { roaring64_bitmap_free(r); return NULL; } - - roaring_bitmap_t *bitmap32 = roaring_bitmap_portable_deserialize_safe( - buf, maxbytes - read_bytes); - if (bitmap32 == NULL) { + size_t bytesread = 0; + bool is_ok = ra_portable_deserialize(&bitmap32->high_low_container, buf, + maxbytes - read_bytes, &bytesread); + if (!is_ok) { + roaring_free(bitmap32); roaring64_bitmap_free(r); return NULL; } - buf += bitmap32_size; - read_bytes += bitmap32_size; + roaring_bitmap_set_copy_on_write(bitmap32, false); + buf += bytesread; + read_bytes += bytesread; // While we don't attempt to validate much, we must ensure that there // is no duplication in the high 48 bits - inserting into the ART @@ -20307,22 +21538,68 @@ size_t roaring64_bitmap_frozen_serialize(const roaring64_bitmap_t *r, return buf - initial_buf; } -static container_t *container_frozen_view(uint8_t typecode, uint32_t elem_count, - const uint64_t **bitsets, - const uint16_t **arrays, - const rle16_t **runs) { +static roaring64_bitmap_t *alloc_frozen_bitmap( + uint64_t capacity, frozen_container_header_t **headers_out) { + if (capacity > SIZE_MAX / sizeof(frozen_container_header_t)) { + return NULL; + } + size_t ptrs = (size_t)capacity * sizeof(container_t *); + size_t codes = (size_t)capacity * sizeof(uint8_t); + size_t headers = (size_t)capacity * sizeof(frozen_container_header_t); + size_t pad = alignof(container_t *) + alignof(frozen_container_header_t); + if (sizeof(roaring64_bitmap_t) > SIZE_MAX - ptrs || + sizeof(roaring64_bitmap_t) + ptrs > SIZE_MAX - codes || + sizeof(roaring64_bitmap_t) + ptrs + codes > SIZE_MAX - headers || + sizeof(roaring64_bitmap_t) + ptrs + codes + headers > SIZE_MAX - pad) { + return NULL; + } + size_t sz = sizeof(roaring64_bitmap_t) + ptrs + codes + headers + pad; + char *base = (char *)roaring_malloc(sz); + if (base == NULL) { + return NULL; + } + char *cursor = base; + roaring64_bitmap_t *r = (roaring64_bitmap_t *)roaring64_arena_alloc( + &cursor, sizeof(roaring64_bitmap_t)); + art_init_cleared(&r->art); + r->flags = ROARING_FLAG_FROZEN; + r->capacity = capacity; + r->first_free = 0; + cursor = roaring64_arena_pad(cursor, base, alignof(container_t *)); + if (capacity == 0) { + r->containers = NULL; + r->typecodes = NULL; + if (headers_out != NULL) { + *headers_out = NULL; + } + return r; + } + r->containers = (container_t **)roaring64_arena_alloc(&cursor, ptrs); + memset(r->containers, 0, ptrs); + r->typecodes = (uint8_t *)roaring64_arena_alloc(&cursor, codes); + cursor = + roaring64_arena_pad(cursor, base, alignof(frozen_container_header_t)); + frozen_container_header_t *hdrs = + (frozen_container_header_t *)roaring64_arena_alloc(&cursor, headers); + if (headers_out != NULL) { + *headers_out = hdrs; + } + return r; +} + +static container_t *container_frozen_view_at( + frozen_container_header_t *header, uint8_t typecode, uint32_t elem_count, + const uint64_t **bitsets, const uint16_t **arrays, const rle16_t **runs) { switch (typecode) { case BITSET_CONTAINER_TYPE: { - bitset_container_t *c = (bitset_container_t *)roaring_malloc( - sizeof(bitset_container_t)); + bitset_container_t *c = &header->bitset; c->cardinality = elem_count; c->words = (uint64_t *)*bitsets; *bitsets += BITSET_CONTAINER_SIZE_IN_WORDS; return (container_t *)c; } case ARRAY_CONTAINER_TYPE: { - array_container_t *c = - (array_container_t *)roaring_malloc(sizeof(array_container_t)); + array_container_t *c = &header->array; c->cardinality = elem_count; c->capacity = elem_count; c->array = (uint16_t *)*arrays; @@ -20330,19 +21607,15 @@ static container_t *container_frozen_view(uint8_t typecode, uint32_t elem_count, return (container_t *)c; } case RUN_CONTAINER_TYPE: { - run_container_t *c = - (run_container_t *)roaring_malloc(sizeof(run_container_t)); + run_container_t *c = &header->run; c->n_runs = elem_count; c->capacity = elem_count; c->runs = (rle16_t *)*runs; *runs += elem_count; return (container_t *)c; } - default: { - assert(false); - roaring_unreachable; + default: return NULL; - } } } @@ -20355,38 +21628,43 @@ roaring64_bitmap_t *roaring64_bitmap_frozen_view(const char *buf, return NULL; } - roaring64_bitmap_t *r = roaring64_bitmap_create(); - - // Flags. - if (maxbytes < sizeof(r->flags)) { - roaring64_bitmap_free(r); + uint8_t flags; + uint64_t capacity; + if (maxbytes < sizeof(flags) + sizeof(capacity)) { return NULL; } - memcpy(&r->flags, buf, sizeof(r->flags)); - buf += sizeof(r->flags); - maxbytes -= sizeof(r->flags); - r->flags |= ROARING_FLAG_FROZEN; - - // Container count. - if (maxbytes < sizeof(r->capacity)) { - roaring64_bitmap_free(r); + memcpy(&flags, buf, sizeof(flags)); + buf += sizeof(flags); + maxbytes -= sizeof(flags); + memcpy(&capacity, buf, sizeof(capacity)); + buf += sizeof(capacity); + maxbytes -= sizeof(capacity); + + // The element counts alone need two bytes per container, so a capacity + // larger than that cannot be satisfied by this buffer. Checked before + // allocating, so a short buffer claiming a huge count cannot make us + // reserve (and clear) an arena sized from attacker-controlled bytes. + if (capacity > maxbytes / sizeof(uint16_t)) { return NULL; } - memcpy(&r->capacity, buf, sizeof(r->capacity)); - buf += sizeof(r->capacity); - maxbytes -= sizeof(r->capacity); - r->containers = - (container_t **)roaring_malloc(r->capacity * sizeof(container_t *)); + frozen_container_header_t *headers = NULL; + roaring64_bitmap_t *r = alloc_frozen_bitmap(capacity, &headers); + if (r == NULL) { + return NULL; + } + // Only flags the format actually defines; the byte comes from the buffer. + r->flags = (uint8_t)(flags & ROARING_FLAG_COW) | ROARING_FLAG_FROZEN | + ROARING_FLAG_FROZEN_ART; // Container element counts. - if (maxbytes < r->capacity * sizeof(uint16_t)) { + if (maxbytes < capacity * sizeof(uint16_t)) { roaring64_bitmap_free(r); return NULL; } const char *elem_counts = buf; - buf += r->capacity * sizeof(uint16_t); - maxbytes -= r->capacity * sizeof(uint16_t); + buf += capacity * sizeof(uint16_t); + maxbytes -= capacity * sizeof(uint16_t); // Total container sizes. uint64_t total_sizes[4]; @@ -20434,6 +21712,10 @@ roaring64_bitmap_t *roaring64_bitmap_frozen_view(const char *buf, // Deserialize in ART iteration order. art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); for (size_t i = 0; it.value != NULL; ++i) { + if (i >= capacity) { + roaring64_bitmap_free(r); + return NULL; + } leaf_t leaf = (leaf_t)*it.value; uint8_t typecode = get_typecode(leaf); @@ -20444,8 +21726,17 @@ roaring64_bitmap_t *roaring64_bitmap_frozen_view(const char *buf, // The container index is unrelated to the iteration order. uint64_t index = get_index(leaf); - r->containers[index] = container_frozen_view(typecode, elem_count, - &bitsets, &arrays, &runs); + if (index >= capacity) { + roaring64_bitmap_free(r); + return NULL; + } + container_t *c = container_frozen_view_at( + headers + index, typecode, elem_count, &bitsets, &arrays, &runs); + if (c == NULL) { + roaring64_bitmap_free(r); + return NULL; + } + set_container_at(r, index, c, typecode); art_iterator_next(&it); } @@ -20453,7 +21744,288 @@ roaring64_bitmap_t *roaring64_bitmap_frozen_view(const char *buf, // Padding to make overall size a multiple of required alignment. buf = CROARING_ALIGN_BUF(buf, CROARING_BITSET_ALIGNMENT); + r->first_free = r->capacity; + return r; +} + +static bool view_one_portable32(roaring64_bitmap_t *r, + frozen_container_header_t *headers, + uint32_t high32, const char *buf, + size_t maxbytes, size_t *consumed) { + *consumed = roaring_bitmap_portable_deserialize_size(buf, maxbytes); + if (*consumed == 0 || *consumed > maxbytes) { + return false; + } + const char *start = buf; + size_t remaining = *consumed; + + uint32_t cookie; + memcpy(&cookie, buf, sizeof(cookie)); + cookie = croaring_letoh32(cookie); + buf += sizeof(cookie); + remaining -= sizeof(cookie); + + int32_t num_containers; + const char *run_flag_bitset = NULL; + bool hasrun = false; + bool has_offsets; + + if (cookie == SERIAL_COOKIE_NO_RUNCONTAINER) { + if (remaining < sizeof(uint32_t)) { + return false; + } + uint32_t n_le; + memcpy(&n_le, buf, sizeof(n_le)); + num_containers = (int32_t)croaring_letoh32(n_le); + buf += sizeof(uint32_t); + remaining -= sizeof(uint32_t); + has_offsets = true; + } else if ((cookie & 0xFFFF) == SERIAL_COOKIE) { + num_containers = (int32_t)(cookie >> 16) + 1; + hasrun = true; + int32_t run_flag_bitset_size = (num_containers + 7) / 8; + if (num_containers < 0 || remaining < (size_t)run_flag_bitset_size) { + return false; + } + run_flag_bitset = buf; + buf += run_flag_bitset_size; + remaining -= (size_t)run_flag_bitset_size; + has_offsets = num_containers >= NO_OFFSET_THRESHOLD; + } else { + return false; + } + if (num_containers < 0 || num_containers > (1 << 16)) { + return false; + } + + size_t desc_bytes = (size_t)num_containers * 2 * sizeof(uint16_t); + if (remaining < desc_bytes) { + return false; + } + const char *keyscards = buf; + buf += desc_bytes; + remaining -= desc_bytes; + + const char *offset_bytes = NULL; + if (has_offsets) { + size_t off_bytes = (size_t)num_containers * sizeof(uint32_t); + if (remaining < off_bytes) { + return false; + } + offset_bytes = buf; + buf += off_bytes; + remaining -= off_bytes; + } + + int32_t last_key = -1; + uint64_t key_base = ((uint64_t)high32) << 32; + + for (int32_t i = 0; i < num_containers; ++i) { + uint16_t key, card_m1; + memcpy(&key, keyscards + 4 * (size_t)i, sizeof(key)); + key = croaring_letoh16(key); + memcpy(&card_m1, keyscards + 4 * (size_t)i + 2, sizeof(card_m1)); + card_m1 = croaring_letoh16(card_m1); + if ((int32_t)key <= last_key) { + return false; + } + last_key = (int32_t)key; + + uint32_t cardinality = (uint32_t)card_m1 + 1; + bool isbitmap = cardinality > DEFAULT_MAX_SIZE; + bool isrun = false; + if (hasrun && (run_flag_bitset[i / 8] & (1 << (i % 8))) != 0) { + isbitmap = false; + isrun = true; + } + + const char *payload; + if (offset_bytes != NULL) { + uint32_t off; + memcpy(&off, offset_bytes + (size_t)i * sizeof(uint32_t), + sizeof(off)); + off = croaring_letoh32(off); + if ((size_t)off >= *consumed) { + return false; + } + payload = start + off; + } else { + payload = buf; + } + + uint8_t typecode; + size_t payload_size; + if (isbitmap) { + typecode = BITSET_CONTAINER_TYPE; + payload_size = BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + } else if (isrun) { + typecode = RUN_CONTAINER_TYPE; + if ((size_t)(payload - start) + sizeof(uint16_t) > *consumed) { + return false; + } + uint16_t n_runs; + memcpy(&n_runs, payload, sizeof(n_runs)); + n_runs = croaring_letoh16(n_runs); + payload_size = sizeof(uint16_t) + (size_t)n_runs * sizeof(rle16_t); + } else { + typecode = ARRAY_CONTAINER_TYPE; + payload_size = (size_t)cardinality * sizeof(uint16_t); + } + if ((size_t)(payload - start) + payload_size > *consumed) { + return false; + } + + if (r->first_free >= r->capacity) { + return false; + } + uint64_t index = allocate_index(r); + frozen_container_header_t *header = headers + index; + container_t *c; + if (isbitmap) { + header->bitset.cardinality = (int32_t)cardinality; + header->bitset.words = (uint64_t *)payload; + c = (container_t *)&header->bitset; + } else if (isrun) { + uint16_t n_runs; + memcpy(&n_runs, payload, sizeof(n_runs)); + n_runs = croaring_letoh16(n_runs); + header->run.n_runs = n_runs; + header->run.capacity = n_runs; + header->run.runs = (rle16_t *)(payload + sizeof(uint16_t)); + c = (container_t *)&header->run; + } else { + header->array.cardinality = (int32_t)cardinality; + header->array.capacity = (int32_t)cardinality; + header->array.array = (uint16_t *)payload; + c = (container_t *)&header->array; + } + set_container_at(r, index, c, typecode); + + uint8_t high48[ART_KEY_BYTES]; + uint64_t high48_bits = key_base | ((uint64_t)key << 16); + split_key(high48_bits, high48); + art_insert(&r->art, high48, (art_val_t)create_leaf(index, typecode)); + + if (offset_bytes == NULL) { + buf += payload_size; + remaining -= payload_size; + } + } + return true; +} + +CROARING_ALLOW_UNALIGNED +roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_frozen( + const char *buf, size_t maxbytes) { + if (buf == NULL) { + return NULL; + } +#if CROARING_IS_BIG_ENDIAN + // The portable format is little-endian and this function uses the payload + // bytes where they sit, so there is no correct view of them here. Refuse + // rather than hand back a bitmap that silently reads byte-swapped values. + (void)maxbytes; + return NULL; +#else + size_t remaining = maxbytes; + + if (remaining < sizeof(uint64_t)) { + return NULL; + } + uint64_t buckets; + memcpy(&buckets, buf, sizeof(buckets)); + buckets = croaring_letoh64(buckets); + buf += sizeof(buckets); + remaining -= sizeof(buckets); + if (buckets > UINT32_MAX) { + return NULL; + } + + uint64_t ncontainers = 0; + const char *count_buf = buf; + size_t count_remaining = remaining; + int64_t previous_high32 = -1; + for (uint64_t bucket = 0; bucket < buckets; ++bucket) { + if (count_remaining < sizeof(uint32_t)) { + return NULL; + } + uint32_t high32; + memcpy(&high32, count_buf, sizeof(high32)); + high32 = croaring_letoh32(high32); + count_buf += sizeof(high32); + count_remaining -= sizeof(high32); + if (high32 <= previous_high32) { + return NULL; + } + previous_high32 = high32; + size_t bitmap32_size = roaring_bitmap_portable_deserialize_size( + count_buf, count_remaining); + if (bitmap32_size == 0) { + return NULL; + } + uint32_t cookie; + if (count_remaining < sizeof(cookie)) { + return NULL; + } + memcpy(&cookie, count_buf, sizeof(cookie)); + cookie = croaring_letoh32(cookie); + int32_t size; + if ((cookie & 0xFFFF) == SERIAL_COOKIE) { + size = (int32_t)(cookie >> 16) + 1; + } else if (cookie == SERIAL_COOKIE_NO_RUNCONTAINER) { + if (count_remaining < 2 * sizeof(uint32_t)) { + return NULL; + } + uint32_t size_le; + memcpy(&size_le, count_buf + sizeof(uint32_t), sizeof(size_le)); + size = (int32_t)croaring_letoh32(size_le); + } else { + return NULL; + } + if (size < 0) { + return NULL; + } + ncontainers += (uint64_t)size; + count_buf += bitmap32_size; + count_remaining -= bitmap32_size; + } + + frozen_container_header_t *headers = NULL; + roaring64_bitmap_t *r = alloc_frozen_bitmap(ncontainers, &headers); + if (r == NULL) { + return NULL; + } + // ART is owned; payloads alias buf. + r->flags = ROARING_FLAG_FROZEN; + + previous_high32 = -1; + for (uint64_t bucket = 0; bucket < buckets; ++bucket) { + if (remaining < sizeof(uint32_t)) { + roaring64_bitmap_free(r); + return NULL; + } + uint32_t high32; + memcpy(&high32, buf, sizeof(high32)); + high32 = croaring_letoh32(high32); + buf += sizeof(high32); + remaining -= sizeof(high32); + if (high32 <= previous_high32) { + roaring64_bitmap_free(r); + return NULL; + } + previous_high32 = high32; + + size_t consumed = 0; + if (!view_one_portable32(r, headers, high32, buf, remaining, + &consumed)) { + roaring64_bitmap_free(r); + return NULL; + } + buf += consumed; + remaining -= consumed; + } return r; +#endif } bool roaring64_bitmap_iterate(const roaring64_bitmap_t *r, @@ -20512,71 +22084,116 @@ roaring64_iterator_t *roaring64_iterator_copy(const roaring64_iterator_t *it) { void roaring64_iterator_free(roaring64_iterator_t *it) { roaring_free(it); } -bool roaring64_iterator_has_value(const roaring64_iterator_t *it) { - return it->has_value; -} +CROARING_STATIC_ASSERT(offsetof(roaring64_iterator_t, pub) == 0, + "the public members must be first in the iterator"); -uint64_t roaring64_iterator_value(const roaring64_iterator_t *it) { - return it->value; +extern inline bool roaring64_iterator_has_value(const roaring64_iterator_t *it); +extern inline uint64_t roaring64_iterator_value(const roaring64_iterator_t *it); + +// The current container is exhausted: step the ART to the next leaf. +static inline bool roaring64_iterator_next_leaf(roaring64_iterator_t *it) { + if (art_iterator_next(&it->art_it)) { + return roaring64_iterator_init_at_leaf_first(it); + } + it->saturated_forward = true; + it->fast_type = 0; + return (it->pub.has_value = false); } -bool roaring64_iterator_advance(roaring64_iterator_t *it) { +// Everything `advance` needs when the bitset cache does not describe where the +// iterator is: a restart, or a step through the container iterator. +static inline bool roaring64_iterator_advance_slow(roaring64_iterator_t *it) { if (it->art_it.value == NULL) { if (it->saturated_forward) { - return (it->has_value = false); + return (it->pub.has_value = false); } roaring64_iterator_init_at(it->r, it, /*first=*/true); - return it->has_value; + return it->pub.has_value; } leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = (uint16_t)it->value; - if (container_iterator_next(get_container(it->r, leaf), get_typecode(leaf), + uint8_t typecode = get_typecode(leaf); + uint16_t low16 = (uint16_t)it->pub.value; + if (container_iterator_next(get_container(it->r, leaf), typecode, &it->container_it, &low16)) { - it->value = it->high48 | low16; - return (it->has_value = true); + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + if (typecode == BITSET_CONTAINER_TYPE) { + // Only reached when the cache was stale, i.e. something else moved + // the cursor. Array and run containers never have a cache, and + // priming them would cost a leaf load per value. + roaring64_iterator_prime(it); + } + return true; } - if (art_iterator_next(&it->art_it)) { - return roaring64_iterator_init_at_leaf_first(it); + return roaring64_iterator_next_leaf(it); +} + +bool roaring64_iterator_advance(roaring64_iterator_t *it) { + // Matching both the leaf and `container_it.index` is enough to know the + // cache describes where the iterator actually is: within a container the + // index determines the position. Anything that moved the cursor -- + // `previous`, `move_equalorlarger`, any of the `read` variants -- moved one + // of them, and drops through to the slow path, which rebuilds the cache. + if (it->fast_type != 0 && it->fast_art_value == it->art_it.value && + it->fast_index == it->container_it.index) { + uint64_t word = it->fast_word; + uint32_t wi = it->fast_wordindex; + while (word == 0) { + if (++wi >= BITSET_CONTAINER_SIZE_IN_WORDS) { + return roaring64_iterator_next_leaf(it); + } + word = it->fast_words[wi]; + it->fast_wordindex = wi; + } + uint32_t index = wi * 64 + (uint32_t)roaring_trailing_zeroes(word); + it->fast_word = word & (word - 1); + it->fast_index = (int32_t)index; + it->container_it.index = (int32_t)index; + it->pub.value = it->high48 | index; + return (it->pub.has_value = true); } - it->saturated_forward = true; - return (it->has_value = false); + return roaring64_iterator_advance_slow(it); } bool roaring64_iterator_previous(roaring64_iterator_t *it) { if (it->art_it.value == NULL) { if (!it->saturated_forward) { // Saturated backward. - return (it->has_value = false); + return (it->pub.has_value = false); } roaring64_iterator_init_at(it->r, it, /*first=*/false); - return it->has_value; + return it->pub.has_value; } leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = (uint16_t)it->value; + uint16_t low16 = (uint16_t)it->pub.value; if (container_iterator_prev(get_container(it->r, leaf), get_typecode(leaf), &it->container_it, &low16)) { - it->value = it->high48 | low16; - return (it->has_value = true); + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + it->fast_type = 0; + return true; } if (art_iterator_prev(&it->art_it)) { return roaring64_iterator_init_at_leaf_last(it); } it->saturated_forward = false; // Saturated backward. - return (it->has_value = false); + it->fast_type = 0; + return (it->pub.has_value = false); } bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, uint64_t val) { uint8_t val_high48[ART_KEY_BYTES]; uint16_t val_low16 = split_key(val, val_high48); - if (!it->has_value || it->high48 != (val & 0xFFFFFFFFFFFF0000)) { + if (!it->pub.has_value || it->high48 != (val & 0xFFFFFFFFFFFF0000)) { // The ART iterator is before or after the high48 bits of `val` (or // beyond the ART altogether), so we need to move to a leaf with a // key equal or greater. if (!art_iterator_lower_bound(&it->art_it, val_high48)) { // Only smaller keys found. it->saturated_forward = true; - return (it->has_value = false); + it->fast_type = 0; + return (it->pub.has_value = false); } it->high48 = combine_key(it->art_it.key, 0); // Fall through to the next if statement. @@ -20586,17 +22203,20 @@ bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, // We're at equal high bits, check if a suitable value can be found // in this container. leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = (uint16_t)it->value; + uint16_t low16 = (uint16_t)it->pub.value; if (container_iterator_lower_bound( get_container(it->r, leaf), get_typecode(leaf), &it->container_it, &low16, val_low16)) { - it->value = it->high48 | low16; - return (it->has_value = true); + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + it->fast_type = 0; + return true; } // Only smaller entries in this container, move to the next. if (!art_iterator_next(&it->art_it)) { it->saturated_forward = true; - return (it->has_value = false); + it->fast_type = 0; + return (it->pub.has_value = false); } } @@ -20608,10 +22228,10 @@ bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, uint64_t roaring64_iterator_read(roaring64_iterator_t *it, uint64_t *buf, uint64_t count) { uint64_t consumed = 0; - while (it->has_value && consumed < count) { + while (it->pub.has_value && consumed < count) { uint32_t container_consumed; leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = (uint16_t)it->value; + uint16_t low16 = (uint16_t)it->pub.value; uint32_t container_count = UINT32_MAX; if (count - consumed < (uint64_t)UINT32_MAX) { container_count = count - consumed; @@ -20622,21 +22242,132 @@ uint64_t roaring64_iterator_read(roaring64_iterator_t *it, uint64_t *buf, consumed += container_consumed; buf += container_consumed; if (has_value) { - it->has_value = true; - it->value = it->high48 | low16; + it->pub.has_value = true; + it->pub.value = it->high48 | low16; + it->fast_type = 0; assert(consumed == count); return consumed; } - it->has_value = art_iterator_next(&it->art_it); - if (it->has_value) { + it->pub.has_value = art_iterator_next(&it->art_it); + if (it->pub.has_value) { roaring64_iterator_init_at_leaf_first(it); } else { it->saturated_forward = true; + it->fast_type = 0; + } + } + return consumed; +} + +uint64_t roaring64_iterator_read_backward(roaring64_iterator_t *it, + uint64_t *buf, uint64_t count) { + uint64_t consumed = 0; + while (it->pub.has_value && consumed < count) { + uint32_t container_consumed; + leaf_t leaf = *it->art_it.value; + uint16_t low16 = (uint16_t)it->pub.value; + uint32_t container_count = UINT32_MAX; + if (count - consumed < (uint64_t)UINT32_MAX) { + container_count = count - consumed; + } + bool has_value = container_iterator_read_backward_into_uint64( + get_container(it->r, leaf), get_typecode(leaf), &it->container_it, + it->high48, buf, container_count, &container_consumed, &low16); + consumed += container_consumed; + buf += container_consumed; + if (has_value) { + it->pub.has_value = true; + it->pub.value = it->high48 | low16; + assert(consumed == count); + return consumed; + } + it->pub.has_value = art_iterator_prev(&it->art_it); + if (it->pub.has_value) { + roaring64_iterator_init_at_leaf_last(it); + } else { + it->saturated_forward = false; } } return consumed; } +size_t roaring64_iterator_read_ranges(roaring64_iterator_t *it, + roaring64_range_closed_t *buf, + size_t count) { + size_t ret = 0; + while (it->pub.has_value && ret < count) { + buf[ret].min = it->pub.value; + for (;;) { + uint16_t low16 = (uint16_t)it->pub.value; + leaf_t leaf = (leaf_t)*it->art_it.value; + bool container_has_more; + uint16_t run_end_low16 = container_iterator_find_run_end( + get_container(it->r, leaf), get_typecode(leaf), + &it->container_it, &low16, &container_has_more); + buf[ret].max = it->high48 | run_end_low16; + + if (container_has_more) { + it->pub.value = it->high48 | low16; + break; + } + // Move to next leaf + it->pub.has_value = art_iterator_next(&it->art_it); + if (it->pub.has_value) { + roaring64_iterator_init_at_leaf_first(it); + } else { + it->saturated_forward = true; + break; + } + // Continue merging only if the run reached the container + // boundary and the next leaf starts exactly at max+1. + if (run_end_low16 != UINT16_MAX || + it->pub.value != buf[ret].max + 1) { + break; + } + } + ret++; + } + return ret; +} + +size_t roaring64_iterator_read_prev_ranges(roaring64_iterator_t *it, + roaring64_range_closed_t *buf, + size_t count) { + size_t ret = 0; + while (it->pub.has_value && ret < count) { + buf[ret].max = it->pub.value; + for (;;) { + uint16_t low16 = (uint16_t)it->pub.value; + leaf_t leaf = (leaf_t)*it->art_it.value; + bool container_has_more; + uint16_t run_start_low16 = container_iterator_find_run_start( + get_container(it->r, leaf), get_typecode(leaf), + &it->container_it, &low16, &container_has_more); + buf[ret].min = it->high48 | run_start_low16; + + if (container_has_more) { + it->pub.value = it->high48 | low16; + break; + } + // Move to previous leaf + it->pub.has_value = art_iterator_prev(&it->art_it); + if (it->pub.has_value) { + roaring64_iterator_init_at_leaf_last(it); + } else { + it->saturated_forward = false; + break; + } + // Continue merging only if the run reached the container + // boundary and the previous leaf ends exactly at min-1. + if (run_start_low16 != 0 || it->pub.value != buf[ret].min - 1) { + break; + } + } + ret++; + } + return ret; +} + #ifdef __cplusplus } // extern "C" } // namespace roaring diff --git a/Sources/SwiftRoaring/Roaring64Bitmap.swift b/Sources/SwiftRoaring/Roaring64Bitmap.swift new file mode 100644 index 0000000..f7cce78 --- /dev/null +++ b/Sources/SwiftRoaring/Roaring64Bitmap.swift @@ -0,0 +1,1380 @@ +import Foundation +import croaring + +/// +/// This structure contains different values about a given Roaring64Bitmap +/// +public typealias Roaring64Statistics = roaring64_statistics_t + +/// +/// Swift wrapper for the 64-bit bitmaps of CRoaring (a C/C++ implementation at +/// https://github.com/RoaringBitmap/CRoaring) +/// +/// A `Roaring64Bitmap` stores `UInt64` values by partitioning the value space on +/// the high 48 bits and using ordinary Roaring containers for the low 16 bits. +/// The API mirrors `RoaringBitmap`, minus the operations CRoaring only offers in +/// 32 bits (the lazy operations, the many-way unions, and the non-portable +/// serialization format). +/// +/// Every function of CRoaring's public 64-bit API is reachable from this type: +/// `roaring64_bitmap_add_offset_signed` through the `addOffset(_:)` and +/// `subtractOffset(_:)` pair, and the variadic `roaring64_bitmap_from` macro +/// through `init(values:)` and array literals. +/// +public final class Roaring64Bitmap: Sequence, Equatable, CustomStringConvertible, + Hashable, ExpressibleByArrayLiteral, SetAlgebra, Codable { + + @usableFromInline + var ptr: OpaquePointer + + /// + /// When this bitmap is a *frozen view* created by `frozenView(bytes:)`, this + /// holds the aligned buffer the view reads from. It is deallocated, after the + /// bitmap itself, by `deinit`. It is `nil` for every ordinary bitmap. + /// + @usableFromInline + var frozenBacking: UnsafeMutableRawPointer? + + public typealias Element = UInt64 + + ///////////////////////////////////////////////////////////////////////////// + /// CONSTRUCTORS /// + ///////////////////////////////////////////////////////////////////////////// + + /// + /// Creates a new bitmap (initially empty) + /// + required public init() { + self.ptr = croaring.roaring64_bitmap_create()! + } + + /// + /// Creates a new bitmap using a given ptr + /// + required public init(ptr: OpaquePointer) { + self.ptr = ptr + } + + /// + /// Add all the values between min (included) and max (excluded) that are at a + /// distance k*step from min. + /// + public init(min: UInt64, max: UInt64, step: UInt64) { + self.ptr = croaring.roaring64_bitmap_from_range(min, max, step) + ?? croaring.roaring64_bitmap_create()! + } + + public convenience init(range: Range, step: UInt64) { + self.init(min: range.lowerBound, max: range.upperBound, step: step) + } + + /// + /// Creates a new bitmap from an array of `uint64_t` integers + /// + public init(values: [UInt64]) { + self.ptr = croaring.roaring64_bitmap_of_ptr(values.count, values)! + } + + /// + /// Creates a new 64-bit bitmap holding the values of a 32-bit one, by moving + /// its containers over. This is cheaper than copying them, but it leaves + /// `bitmap` empty. + /// + public init(moving bitmap: RoaringBitmap) { + self.ptr = croaring.roaring64_bitmap_move_from_roaring32(bitmap.ptr)! + } + + /// + /// Creates a new 64-bit bitmap holding the values of a 32-bit one, which is + /// left unchanged. + /// + public convenience init(_ bitmap: RoaringBitmap) { + self.init(moving: bitmap.copy()) + } + + public required init(arrayLiteral: Element...) { + self.ptr = croaring.roaring64_bitmap_of_ptr(arrayLiteral.count, arrayLiteral)! + } + + deinit { + croaring.roaring64_bitmap_free(self.ptr) + self.frozenBacking?.deallocate() + } + + ///////////////////////////////////////////////////////////////////////////// + /// OPERATORS /// + ///////////////////////////////////////////////////////////////////////////// + + /// + /// Computes the intersection between two bitmaps and returns new bitmap. + /// + @inlinable @inline(__always) + public func intersection(_ other: Roaring64Bitmap) -> Self { + return Self(ptr: croaring.roaring64_bitmap_and(self.ptr, other.ptr)) + } + /// + /// Computes the intersection between two bitmaps and returns new bitmap. + /// + @inlinable @inline(__always) + public static func &(lhs: Roaring64Bitmap, rhs: Roaring64Bitmap) -> Self { + return Self(ptr: croaring.roaring64_bitmap_and(lhs.ptr, rhs.ptr)) + } + + /// + /// Inplace version modifies x1, x1 == x2 is allowed + /// + @inlinable @inline(__always) + public func formIntersection(_ other: Roaring64Bitmap) { + croaring.roaring64_bitmap_and_inplace(self.ptr, other.ptr) + } + /// + /// Inplace version modifies x1, x1 == x2 is allowed + /// + @inlinable @inline(__always) + public static func &=(lhs: Roaring64Bitmap, rhs: Roaring64Bitmap) { + lhs.formIntersection(rhs) + } + + /// + /// Computes the size of the intersection between two bitmaps. + /// + @inlinable @inline(__always) + public func intersectionCount(_ other: Roaring64Bitmap) -> UInt64 { + return croaring.roaring64_bitmap_and_cardinality(self.ptr, other.ptr) + } + + /// + /// Check whether two bitmaps intersect. + /// + @inlinable @inline(__always) + public func intersect(_ other: Roaring64Bitmap) -> Bool { + return croaring.roaring64_bitmap_intersect(self.ptr, other.ptr) + } + + /// + /// Check whether the bitmap has any value in the range [min, max). + /// + @inlinable @inline(__always) + public func intersectWithRange(min: UInt64, max: UInt64) -> Bool { + return croaring.roaring64_bitmap_intersect_with_range(self.ptr, min, max) + } + + @inlinable @inline(__always) + public func intersects(_ range: Range) -> Bool { + return croaring.roaring64_bitmap_intersect_with_range( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// Computes the Jaccard index between two bitmaps. (Also known as the Tanimoto + /// distance, or the Jaccard similarity coefficient). + /// + /// The Jaccard index is undefined if both bitmaps are empty. + /// + @inlinable @inline(__always) + public func jaccardIndex(_ other: Roaring64Bitmap) -> Double { + return croaring.roaring64_bitmap_jaccard_index(self.ptr, other.ptr) + } + + /// + /// Computes the size of the union between two bitmaps. + /// + @inlinable @inline(__always) + public func unionCount(_ other: Roaring64Bitmap) -> UInt64 { + return croaring.roaring64_bitmap_or_cardinality(self.ptr, other.ptr) + } + + /// + /// Computes the size of the difference (andnot) between two bitmaps. + /// + @inlinable @inline(__always) + public func subtractingCount(_ other: Roaring64Bitmap) -> UInt64 { + return croaring.roaring64_bitmap_andnot_cardinality(self.ptr, other.ptr) + } + + /// + /// Computes the size of the symmetric difference (xor) between two bitmaps. + /// + @inlinable @inline(__always) + public func symmetricDifferenceCount(_ other: Roaring64Bitmap) -> UInt64 { + return croaring.roaring64_bitmap_xor_cardinality(self.ptr, other.ptr) + } + + /// + /// Returns the number of elements in range [min, max). + /// + @inlinable @inline(__always) + public func rangeCardinality(min: UInt64, max: UInt64) -> UInt64 { + return croaring.roaring64_bitmap_range_cardinality(self.ptr, min, max) + } + + /// + /// Returns the number of elements in range [min, max]. + /// + @inlinable @inline(__always) + public func rangeCardinalityClosed(min: UInt64, max: UInt64) -> UInt64 { + return croaring.roaring64_bitmap_range_closed_cardinality(self.ptr, min, max) + } + + @inlinable @inline(__always) + public func cardinality(in range: Range) -> UInt64 { + return croaring.roaring64_bitmap_range_cardinality( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + @inlinable @inline(__always) + public func cardinality(in range: ClosedRange) -> UInt64 { + return croaring.roaring64_bitmap_range_closed_cardinality( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// Computes the union between two bitmaps and returns new bitmap. + /// + @inlinable @inline(__always) + public func union(_ other: Roaring64Bitmap) -> Self { + return Self(ptr: croaring.roaring64_bitmap_or(self.ptr, other.ptr)) + } + /// + /// Computes the union between two bitmaps and returns new bitmap. + /// + @inlinable @inline(__always) + public static func |(lhs: Roaring64Bitmap, rhs: Roaring64Bitmap) -> Self { + return Self(ptr: croaring.roaring64_bitmap_or(lhs.ptr, rhs.ptr)) + } + + /// + /// Inplace version of `roaring64_bitmap_or`, modifies x1. + /// + @inlinable @inline(__always) + public func formUnion(_ other: Roaring64Bitmap) { + croaring.roaring64_bitmap_or_inplace(self.ptr, other.ptr) + } + /// + /// Inplace version of `roaring64_bitmap_or`, modifies x1. + /// + @inlinable @inline(__always) + public static func |=(lhs: Roaring64Bitmap, rhs: Roaring64Bitmap) { + lhs.formUnion(rhs) + } + + /// + /// Computes the symmetric difference (xor) between two bitmaps + /// and returns new bitmap. + /// + @inlinable @inline(__always) + public func symmetricDifference(_ other: Roaring64Bitmap) -> Self { + return Self(ptr: croaring.roaring64_bitmap_xor(self.ptr, other.ptr)) + } + /// + /// Computes the symmetric difference (xor) between two bitmaps + /// and returns new bitmap. + /// + @inlinable @inline(__always) + public static func ^(lhs: Roaring64Bitmap, rhs: Roaring64Bitmap) -> Self { + return Self(ptr: croaring.roaring64_bitmap_xor(lhs.ptr, rhs.ptr)) + } + + /// + /// Inplace version of `roaring64_bitmap_xor`, modifies x1. x1 != x2. + /// + @inlinable @inline(__always) + public func formSymmetricDifference(_ other: Roaring64Bitmap) { + croaring.roaring64_bitmap_xor_inplace(self.ptr, other.ptr) + } + /// + /// Inplace version of `roaring64_bitmap_xor`, modifies x1. x1 != x2. + /// + @inlinable @inline(__always) + public static func ^=(lhs: Roaring64Bitmap, rhs: Roaring64Bitmap) { + lhs.formSymmetricDifference(rhs) + } + + /// + /// Computes the difference (andnot) between two bitmaps and returns new bitmap. + /// + @inlinable @inline(__always) + public func subtracting(_ other: Roaring64Bitmap) -> Self { + return Self(ptr: croaring.roaring64_bitmap_andnot(self.ptr, other.ptr)) + } + /// + /// Computes the difference (andnot) between two bitmaps and returns new bitmap. + /// + @inlinable @inline(__always) + public static func -(lhs: Roaring64Bitmap, rhs: Roaring64Bitmap) -> Self { + return Self(ptr: croaring.roaring64_bitmap_andnot(lhs.ptr, rhs.ptr)) + } + + /// + /// Inplace version of `roaring64_bitmap_andnot`, modifies x1. x1 != x2. + /// + @inlinable @inline(__always) + public func subtract(_ other: Roaring64Bitmap) { + croaring.roaring64_bitmap_andnot_inplace(self.ptr, other.ptr) + } + /// + /// Inplace version of `roaring64_bitmap_andnot`, modifies x1. x1 != x2. + /// + @inlinable @inline(__always) + public static func -=(lhs: Roaring64Bitmap, rhs: Roaring64Bitmap) { + lhs.subtract(rhs) + } + + /// + /// Return true if the two bitmaps contain the same elements. + /// + @inlinable @inline(__always) + public static func ==(lhs: Roaring64Bitmap, rhs: Roaring64Bitmap) -> Bool { + return croaring.roaring64_bitmap_equals(lhs.ptr, rhs.ptr) + } + + /// + /// Return true if all the elements of ra1 are also in ra2. + /// + @inlinable @inline(__always) + public func isSubset(of other: Roaring64Bitmap) -> Bool { + return croaring.roaring64_bitmap_is_subset(self.ptr, other.ptr) + } + + /// + /// Return true if all the elements of ra1 are also in ra2 and ra2 is strictly + /// greater than ra1. + /// + @inlinable @inline(__always) + public func isStrictSubset(of other: Roaring64Bitmap) -> Bool { + return croaring.roaring64_bitmap_is_strict_subset(self.ptr, other.ptr) + } + + /// + /// Return true if ra1 and ra2 have no elements in common. + /// + @inlinable @inline(__always) + public func isDisjoint(with other: Roaring64Bitmap) -> Bool { + return !croaring.roaring64_bitmap_intersect(self.ptr, other.ptr) + } + + /// + /// compute the negation of the roaring bitmap within a specified + /// interval: [rangeStart, rangeEnd). The number of negated values is + /// rangeEnd - rangeStart. + /// Areas outside the range are passed through unchanged. + /// + @inlinable @inline(__always) + public func flip(rangeStart: UInt64, rangeEnd: UInt64) -> Self { + return Self(ptr: croaring.roaring64_bitmap_flip(self.ptr, rangeStart, rangeEnd)) + } + + @inlinable @inline(__always) + public func flip(_ range: Range) -> Self { + return Self(ptr: croaring.roaring64_bitmap_flip( + self.ptr, + range.lowerBound, + range.upperBound + )) + } + + /// + /// compute the negation of the roaring bitmap within a specified closed + /// interval: [rangeStart, rangeEnd]. + /// Areas outside the range are passed through unchanged. + /// + @inlinable @inline(__always) + public func flipClosed(rangeStart: UInt64, rangeEnd: UInt64) -> Self { + return Self(ptr: croaring.roaring64_bitmap_flip_closed(self.ptr, rangeStart, rangeEnd)) + } + + @inlinable @inline(__always) + public func flip(_ range: ClosedRange) -> Self { + return Self(ptr: croaring.roaring64_bitmap_flip_closed( + self.ptr, + range.lowerBound, + range.upperBound + )) + } + + /// + /// compute (in place) the negation of the roaring bitmap within a specified + /// interval: [rangeStart, rangeEnd). + /// Areas outside the range are passed through unchanged. + /// + @inlinable @inline(__always) + public func flipInplace(rangeStart: UInt64, rangeEnd: UInt64) { + croaring.roaring64_bitmap_flip_inplace(self.ptr, rangeStart, rangeEnd) + } + + @inlinable @inline(__always) + public func flipInPlace(_ range: Range) { + croaring.roaring64_bitmap_flip_inplace( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// compute (in place) the negation of the roaring bitmap within a specified + /// closed interval: [rangeStart, rangeEnd]. + /// Areas outside the range are passed through unchanged. + /// + @inlinable @inline(__always) + public func flipInplaceClosed(rangeStart: UInt64, rangeEnd: UInt64) { + croaring.roaring64_bitmap_flip_closed_inplace(self.ptr, rangeStart, rangeEnd) + } + + @inlinable @inline(__always) + public func flipInPlace(_ range: ClosedRange) { + croaring.roaring64_bitmap_flip_closed_inplace( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// Returns a new bitmap containing every value of this one shifted up by + /// `offset`. Values that would overflow are dropped. + /// + @inlinable @inline(__always) + public func addOffset(_ offset: UInt64) -> Self { + return Self(ptr: croaring.roaring64_bitmap_add_offset(self.ptr, offset)) + } + + /// + /// Returns a new bitmap containing every value of this one shifted down by + /// `offset`. Values that would underflow are dropped. + /// + @inlinable @inline(__always) + public func subtractOffset(_ offset: UInt64) -> Self { + return Self(ptr: croaring.roaring64_bitmap_sub_offset(self.ptr, offset)) + } + + ///////////////////////////////////////////////////////////////////////////// + /// MODIFYING AND QUERYING /// + ///////////////////////////////////////////////////////////////////////////// + + /// + /// Copies a bitmap. This does memory allocation. + /// + @inlinable @inline(__always) + public func copy() -> Self { + return Self(ptr: croaring.roaring64_bitmap_copy(self.ptr)) + } + + /// + /// Replaces the content of this bitmap with a copy of `other`. + /// + /// It might be preferable and simpler to call `copy()`, except that + /// `overwrite(with:)` can save on memory allocations. + /// + @inlinable @inline(__always) + public func overwrite(with other: Roaring64Bitmap) { + croaring.roaring64_bitmap_overwrite(self.ptr, other.ptr) + } + + /// + /// Add value x + /// + @inlinable @inline(__always) + public func add(_ value: UInt64) { + croaring.roaring64_bitmap_add(self.ptr, value) + } + + /// + /// Add all the values of `values`, faster than repeatedly calling `add(_:)` + /// + @inlinable @inline(__always) + public func addMany(values: [UInt64]) { + croaring.roaring64_bitmap_add_many(self.ptr, values.count, values) + } + + /// + /// Add value x + /// Returns true if a new value was added, false if the value was already existing. + /// + @inlinable @inline(__always) + public func addCheck(_ value: UInt64) -> Bool { + return croaring.roaring64_bitmap_add_checked(self.ptr, value) + } + + /// + /// Add all values in range [min, max] + /// + @inlinable @inline(__always) + public func addRangeClosed(min: UInt64, max: UInt64) { + croaring.roaring64_bitmap_add_range_closed(self.ptr, min, max) + } + + @inlinable @inline(__always) + public func add(_ range: ClosedRange) { + croaring.roaring64_bitmap_add_range_closed( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// Add all values in range [min, max) + /// + @inlinable @inline(__always) + public func addRange(min: UInt64, max: UInt64) { + croaring.roaring64_bitmap_add_range(self.ptr, min, max) + } + + @inlinable @inline(__always) + public func add(_ range: Range) { + croaring.roaring64_bitmap_add_range( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + @inlinable @inline(__always) + @discardableResult + public func insert(_ newMember: UInt64) -> (inserted: Bool, memberAfterInsert: UInt64) { + let inserted = self.addCheck(newMember) + return (inserted, newMember) + } + + @inlinable @inline(__always) + @discardableResult + public func update(with newMember: UInt64) -> UInt64? { + guard self.addCheck(newMember) else { return newMember } + return nil + } + + /// + /// Remove value x, and return it if it was there. + /// + @inlinable @inline(__always) + @discardableResult + public func remove(_ value: UInt64) -> UInt64? { + guard self.removeCheck(value) else { return nil } + return value + } + + /// + /// Remove value x, without reporting whether it was there. Slightly faster + /// than `remove(_:)`. + /// + @inlinable @inline(__always) + public func discard(_ value: UInt64) { + croaring.roaring64_bitmap_remove(self.ptr, value) + } + + /// + /// Remove value x + /// Returns true if a value was removed, false if the value was not existing. + /// + @inlinable @inline(__always) + public func removeCheck(_ value: UInt64) -> Bool { + return croaring.roaring64_bitmap_remove_checked(self.ptr, value) + } + + /// + /// Remove multiple values, faster than repeatedly calling `remove(_:)` + /// + @inlinable @inline(__always) + public func removeMany(values: [UInt64]) { + croaring.roaring64_bitmap_remove_many(self.ptr, values.count, values) + } + + /// + /// Remove all values in range [min, max] + /// + @inlinable @inline(__always) + public func removeRangeClosed(min: UInt64, max: UInt64) { + croaring.roaring64_bitmap_remove_range_closed(self.ptr, min, max) + } + + @inlinable @inline(__always) + public func remove(_ range: ClosedRange) { + croaring.roaring64_bitmap_remove_range_closed( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// Remove all values in range [min, max) + /// + @inlinable @inline(__always) + public func removeRange(min: UInt64, max: UInt64) { + croaring.roaring64_bitmap_remove_range(self.ptr, min, max) + } + + @inlinable @inline(__always) + public func remove(_ range: Range) { + croaring.roaring64_bitmap_remove_range( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// Empties the bitmap. + /// + @inlinable @inline(__always) + public func clear() { + croaring.roaring64_bitmap_clear(self.ptr) + } + + @inlinable @inline(__always) + public func removeAll() { + self.clear() + } + + @inlinable @inline(__always) + public func removeAll(where shouldBeRemoved: (UInt64) -> Bool) { + for i in self where shouldBeRemoved(i) { + self.remove(i) + } + } + + /// + /// Get the cardinality of the bitmap (number of elements). + /// + @inlinable @inline(__always) + public var count: UInt64 { + return croaring.roaring64_bitmap_get_cardinality(self.ptr) + } + + /// + /// Check if value x is present + /// + @inlinable @inline(__always) + public func contains(_ value: UInt64) -> Bool { + return croaring.roaring64_bitmap_contains(self.ptr, value) + } + + /// + /// Check whether all the values from `start` (included) to `end` (excluded) + /// are present. + /// + @inlinable @inline(__always) + public func containsRange(start: UInt64, end: UInt64) -> Bool { + return croaring.roaring64_bitmap_contains_range(self.ptr, start, end) + } + + @inlinable @inline(__always) + public func contains(_ range: Range) -> Bool { + return croaring.roaring64_bitmap_contains_range( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// Check whether all the values from `min` to `max` (both included) are present. + /// + @inlinable @inline(__always) + public func containsRangeClosed(min: UInt64, max: UInt64) -> Bool { + return croaring.roaring64_bitmap_contains_range_closed(self.ptr, min, max) + } + + @inlinable @inline(__always) + public func contains(_ range: ClosedRange) -> Bool { + return croaring.roaring64_bitmap_contains_range_closed( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// Check whether the bitmap is empty + /// + @inlinable @inline(__always) + public var isEmpty: Bool { + return croaring.roaring64_bitmap_is_empty(self.ptr) + } + + /// + /// Returns the element having the designated rank, where the smallest element + /// has rank 0. Returns nil if the bitmap holds `rank` elements or fewer. + /// + @inlinable @inline(__always) + public func select(rank: UInt64) -> UInt64? { + var value: UInt64 = 0 + guard croaring.roaring64_bitmap_select(self.ptr, rank, &value) else { return nil } + return value + } + + /// + /// Returns the number of integers that are smaller or equal to x. + /// + @inlinable @inline(__always) + public func rank(value: UInt64) -> UInt64 { + return croaring.roaring64_bitmap_rank(self.ptr, value) + } + + /// + /// Returns the index of `value` in the sorted set, or nil when `value` is + /// not present. It is equivalent to, but faster than, `rank(value:) - 1`. + /// + @inlinable @inline(__always) + public func index(of value: UInt64) -> UInt64? { + var index: UInt64 = 0 + guard croaring.roaring64_bitmap_get_index(self.ptr, value, &index) else { return nil } + return index + } + + /// + /// Returns the smallest value in the set. + /// Returns nil if the set is empty. + /// + @inlinable @inline(__always) + public func min() -> UInt64? { + guard !self.isEmpty else { return nil } + return croaring.roaring64_bitmap_minimum(self.ptr) + } + + @inlinable @inline(__always) + public var first: UInt64? { + return self.min() + } + + @inlinable @inline(__always) + public func popFirst() -> UInt64? { + guard let first = self.first else { return nil } + self.remove(first) + return first + } + + /// + /// Returns the greatest value in the set. + /// Returns nil if the set is empty. + /// + @inlinable @inline(__always) + public func max() -> UInt64? { + guard !self.isEmpty else { return nil } + return croaring.roaring64_bitmap_maximum(self.ptr) + } + + @inlinable @inline(__always) + public var last: UInt64? { + return self.max() + } + + @inlinable @inline(__always) + public func popLast() -> UInt64? { + guard let last = self.last else { return nil } + self.remove(last) + return last + } + + ///////////////////////////////////////////////////////////////////////////// + /// OPTIMIZATION /// + ///////////////////////////////////////////////////////////////////////////// + + /// + /// Remove run-length encoding even when it is more space efficient. + /// Returns whether a change was applied. + /// + @inlinable @inline(__always) + public func removeRunCompression() -> Bool { + return croaring.roaring64_bitmap_remove_run_compression(self.ptr) + } + + /// + /// convert array and bitmap containers to run containers when it is more + /// efficient; also convert from run containers when more space efficient. + /// Returns true if the result has at least one run container. + /// Additional savings might be possible by calling shrink(). + /// + @inlinable @inline(__always) + public func runOptimize() -> Bool { + return croaring.roaring64_bitmap_run_optimize(self.ptr) + } + + /// + /// If needed, reallocate memory to shrink the memory usage. Returns + /// the number of bytes saved. + /// + @inlinable @inline(__always) + public func shrink() -> size_t { + return croaring.roaring64_bitmap_shrink_to_fit(self.ptr) + } + + /// + /// (For advanced users.) + /// Collect statistics about the bitmap. + /// + public func statistics() -> Roaring64Statistics { + var stats = Roaring64Statistics() + croaring.roaring64_bitmap_statistics(self.ptr, &stats) + return stats + } + + /// + /// (For advanced users.) + /// Checks the internal invariants of the bitmap, which is mostly useful on + /// bitmaps that came from an untrusted source. Returns nil when the bitmap + /// is well formed, and the reason it is not otherwise. + /// + public func internalValidate() -> String? { + var reason: UnsafePointer? + guard !croaring.roaring64_bitmap_internal_validate(self.ptr, &reason) else { + return nil + } + return reason.map { String(cString: $0) } ?? "unknown reason" + } + + ///////////////////////////////////////////////////////////////////////////// + /// CONVERSION /// + ///////////////////////////////////////////////////////////////////////////// + + /// + /// Convert the bitmap to an array. + /// + public func toArray() -> [UInt64] { + var array = [UInt64](repeating: 0, count: Int(self.count)) + array.withUnsafeMutableBufferPointer { + croaring.roaring64_bitmap_to_uint64_array(self.ptr, $0.baseAddress) + } + return array + } + + /// + /// Calls `body` once for every value of the bitmap, in increasing order, + /// stopping early as soon as `body` returns false. Returns true if the + /// whole bitmap was visited. + /// + /// This is faster than the `Sequence` conformance, at the cost of not being + /// able to interleave the iteration with other work. + /// + @discardableResult + public func iterate(_ body: (UInt64) -> Bool) -> Bool { + return withoutActuallyEscaping(body) { body in + var body = body + return withUnsafeMutablePointer(to: &body) { context in + croaring.roaring64_bitmap_iterate(self.ptr, { value, context in + let body = context!.assumingMemoryBound(to: ((UInt64) -> Bool).self) + return body.pointee(value) + }, UnsafeMutableRawPointer(context)) + } + } + } + + ///////////////////////////////////////////////////////////////////////////// + /// ITERATION /// + ///////////////////////////////////////////////////////////////////////////// + + /// + /// Creates a Roaring64BitmapIterator positioned on the smallest value. + /// + @inlinable @inline(__always) + public func makeIterator() -> Roaring64BitmapIterator { + return Roaring64BitmapIterator(bitmap: self) + } + + /// + /// Creates a Roaring64BitmapIterator positioned on the largest value, to be + /// walked backward with `previous()` or `nextBackward()`. + /// + @inlinable @inline(__always) + public func makeLastIterator() -> Roaring64BitmapIterator { + return Roaring64BitmapIterator(bitmap: self, atLast: true) + } + + /// + /// Used to iterate through values in a 64-bit roaring bitmap. + /// + /// Unlike its 32-bit counterpart this is a reference type, because CRoaring + /// only hands out heap-allocated 64-bit iterators; use `copy()` to fork one. + /// + public final class Roaring64BitmapIterator: IteratorProtocol { + @usableFromInline + internal var it: OpaquePointer + + /// + /// Keeps the bitmap the iterator reads from alive. + /// + @usableFromInline + internal let bitmap: Roaring64Bitmap + + @inlinable @inline(__always) + internal init(bitmap: Roaring64Bitmap) { + self.it = croaring.roaring64_iterator_create(bitmap.ptr)! + self.bitmap = bitmap + } + + @inlinable @inline(__always) + internal init(bitmap: Roaring64Bitmap, atLast: Bool) { + self.it = atLast + ? croaring.roaring64_iterator_create_last(bitmap.ptr)! + : croaring.roaring64_iterator_create(bitmap.ptr)! + self.bitmap = bitmap + } + + @inlinable @inline(__always) + internal init( + it: OpaquePointer, + bitmap: Roaring64Bitmap + ) { + self.it = it + self.bitmap = bitmap + } + + deinit { + croaring.roaring64_iterator_free(self.it) + } + + /// + /// Returns an independent iterator at the same position. + /// + @inlinable @inline(__always) + public func copy() -> Roaring64BitmapIterator { + return Roaring64BitmapIterator( + it: croaring.roaring64_iterator_copy(self.it)!, + bitmap: self.bitmap + ) + } + + /// + /// Rewinds the iterator to the smallest value of `bitmap`. + /// + @inlinable @inline(__always) + public func reinit() { + croaring.roaring64_iterator_reinit(self.bitmap.ptr, self.it) + } + + /// + /// Moves the iterator to the largest value of `bitmap`. + /// + @inlinable @inline(__always) + public func reinitLast() { + croaring.roaring64_iterator_reinit_last(self.bitmap.ptr, self.it) + } + + /// + /// Whether the iterator currently points at a value. + /// + @inlinable @inline(__always) + public var hasValue: Bool { + return croaring.roaring64_iterator_has_value(self.it) + } + + /// + /// The value the iterator points at, undefined when `hasValue` is false. + /// + @inlinable @inline(__always) + public var currentValue: UInt64 { + return croaring.roaring64_iterator_value(self.it) + } + + /// + /// The value the iterator points at, or nil if it is exhausted. + /// + @inlinable @inline(__always) + public var value: UInt64? { + return self.hasValue ? self.currentValue : nil + } + + @inlinable @inline(__always) + public func next() -> UInt64? { + guard croaring.roaring64_iterator_has_value(self.it) else { return nil } + let value = croaring.roaring64_iterator_value(self.it) + croaring.roaring64_iterator_advance(self.it) + return value + } + + /// + /// Moves the iterator to the next value. Returns true if it still points + /// at a value afterward. + /// + @inlinable @inline(__always) + @discardableResult + public func advance() -> Bool { + return croaring.roaring64_iterator_advance(self.it) + } + + /// + /// Moves the iterator to the previous value. Returns true if it still + /// points at a value afterward. + /// + @inlinable @inline(__always) + @discardableResult + public func previous() -> Bool { + return croaring.roaring64_iterator_previous(self.it) + } + + /// + /// Returns the previous value, walking the bitmap backward, or nil once + /// the iterator is exhausted. Pair with `makeLastIterator()`. + /// + @inlinable @inline(__always) + public func nextBackward() -> UInt64? { + guard croaring.roaring64_iterator_has_value(self.it) else { return nil } + let value = croaring.roaring64_iterator_value(self.it) + croaring.roaring64_iterator_previous(self.it) + return value + } + + /// + /// Moves the iterator to the smallest value that is greater than or equal + /// to `value`. Returns true if such a value exists. + /// + @inlinable @inline(__always) + @discardableResult + public func moveEqualOrLarger(_ value: UInt64) -> Bool { + return croaring.roaring64_iterator_move_equalorlarger(self.it, value) + } + + /// + /// Reads up to `count` values into an array, advancing the iterator past + /// them. The array is shorter than `count` when the bitmap runs out. + /// + public func read(count: UInt64) -> [UInt64] { + var buffer = [UInt64](repeating: 0, count: Int(count)) + let read = buffer.withUnsafeMutableBufferPointer { buf in + croaring.roaring64_iterator_read(self.it, buf.baseAddress, count) + } + buffer.removeLast(Int(count) - Int(read)) + return buffer + } + + /// + /// Reads up to `count` values in decreasing order into an array, moving + /// the iterator backward past them. Pair with `makeLastIterator()`. + /// + public func readBackward(count: UInt64) -> [UInt64] { + var buffer = [UInt64](repeating: 0, count: Int(count)) + let read = buffer.withUnsafeMutableBufferPointer { buf in + croaring.roaring64_iterator_read_backward(self.it, buf.baseAddress, count) + } + buffer.removeLast(Int(count) - Int(read)) + return buffer + } + + /// + /// Reads up to `count` maximal runs of consecutive values, advancing the + /// iterator past them. `{1, 2, 3, 5, 6}` reads as `[1...3, 5...6]`. + /// + public func readRanges(count: Int) -> [ClosedRange] { + var buffer = [roaring64_range_closed_t]( + repeating: roaring64_range_closed_t(), count: count + ) + let read = buffer.withUnsafeMutableBufferPointer { buf in + croaring.roaring64_iterator_read_ranges(self.it, buf.baseAddress, count) + } + return buffer.prefix(read).map { $0.min...$0.max } + } + + /// + /// Reads up to `count` maximal runs of consecutive values in decreasing + /// order, moving the iterator backward past them. + /// + public func readPreviousRanges(count: Int) -> [ClosedRange] { + var buffer = [roaring64_range_closed_t]( + repeating: roaring64_range_closed_t(), count: count + ) + let read = buffer.withUnsafeMutableBufferPointer { buf in + croaring.roaring64_iterator_read_prev_ranges(self.it, buf.baseAddress, count) + } + return buffer.prefix(read).map { $0.min...$0.max } + } + } + + ///////////////////////////////////////////////////////////////////////////// + /// DESCRIPTION /// + ///////////////////////////////////////////////////////////////////////////// + + /// + /// Returns a string representation of the bitset + /// + public var description: String { + var ret = prefix(100).map { $0.description }.joined(separator: ", ") + if self.count >= 100 { + ret.append(", ...") + } + return "Roaring64Bitmap(\(ret))" + } + + /// + /// Returns a hash value for the bitset. This is expensive and should be + /// buffered for performance. + /// + public func hash(into hasher: inout Hasher) { + let b: UInt64 = 31 + var hash: UInt64 = 0 + for i in self { + hash = hash &* b &+ i + } + hasher.combine(hash) + } +} + +///////////////////////////////////////////////////////////////////////////// +/// BULK (CONTEXT) OPERATIONS /// +///////////////////////////////////////////////////////////////////////////// + +extension Roaring64Bitmap { + /// + /// A bit of context usable with the `*Bulk` operations. + /// + /// Reusing a context across calls whose values are in increasing order lets + /// CRoaring skip the container lookup. A context may only be used with a + /// single bitmap, and any modification of that bitmap that does not go + /// through a `*Bulk` call taking the same context invalidates it. + /// + @frozen + public struct BulkContext { + @usableFromInline + internal var context: roaring64_bulk_context_t + + @inlinable @inline(__always) + public init() { + self.context = roaring64_bulk_context_t() + } + + /// + /// Forgets the cached container, making the context usable again after + /// the bitmap was modified out of band (or with another bitmap). + /// + @inlinable @inline(__always) + public mutating func reset() { + self.context = roaring64_bulk_context_t() + } + } + + /// + /// Add value x, using context from a previous insert for speed optimization. + /// + @inlinable @inline(__always) + public func addBulk(_ value: UInt64, context: inout BulkContext) { + croaring.roaring64_bitmap_add_bulk(self.ptr, &context.context, value) + } + + /// + /// Remove value x, using context from a previous removal for speed optimization. + /// + @inlinable @inline(__always) + public func removeBulk(_ value: UInt64, context: inout BulkContext) { + croaring.roaring64_bitmap_remove_bulk(self.ptr, &context.context, value) + } + + /// + /// Check if value x is present, using context from a previous lookup for + /// speed optimization. + /// + @inlinable @inline(__always) + public func containsBulk(_ value: UInt64, context: inout BulkContext) -> Bool { + return croaring.roaring64_bitmap_contains_bulk(self.ptr, &context.context, value) + } +} + +///////////////////////////////////////////////////////////////////////////// +/// SERIALIZATION /// +///////////////////////////////////////////////////////////////////////////// + +extension Roaring64Bitmap { + /// + /// How many bytes are required to serialize this bitmap in the portable + /// format, which is compatible with the Java and Go versions. See the format + /// specification at https://github.com/RoaringBitmap/RoaringFormatSpec + /// + @inlinable @inline(__always) + public func portableSizeInBytes() -> size_t { + return croaring.roaring64_bitmap_portable_size_in_bytes(self.ptr) + } + + /// + /// write a bitmap to a char buffer. The output buffer should refer to at least + /// `portableSizeInBytes()` bytes of allocated memory. Returns how many bytes + /// were written. + /// + @inlinable @inline(__always) + public func portableSerialize(buffer: inout [Int8]) -> size_t { + return croaring.roaring64_bitmap_portable_serialize(self.ptr, &buffer) + } + + /// + /// Serializes the bitmap in the portable format and returns the bytes. + /// + public func portableSerialize() -> [UInt8] { + var buffer = [UInt8](repeating: 0, count: self.portableSizeInBytes()) + guard !buffer.isEmpty else { return buffer } + buffer.withUnsafeMutableBytes { + _ = croaring.roaring64_bitmap_portable_serialize( + self.ptr, + $0.baseAddress!.assumingMemoryBound(to: CChar.self) + ) + } + return buffer + } + + /// + /// read a bitmap from a serialized version in a safe manner (reading up to + /// `maxbytes`). Returns nil if the buffer does not hold a valid bitmap. + /// + public static func portableDeserializeSafe(buffer: [Int8], maxbytes: size_t) -> Self? { + guard let ptr = croaring.roaring64_bitmap_portable_deserialize_safe(buffer, maxbytes) else { + return nil + } + return Self(ptr: ptr) + } + + /// + /// read a bitmap from bytes written by `portableSerialize()`. Returns nil if + /// the bytes do not hold a valid bitmap. + /// + public static func portableDeserializeSafe(bytes: [UInt8]) -> Self? { + let ptr: OpaquePointer? = bytes.withUnsafeBytes { + croaring.roaring64_bitmap_portable_deserialize_safe( + $0.baseAddress?.assumingMemoryBound(to: CChar.self), + bytes.count + ) + } + return ptr.map { Self(ptr: $0) } + } + + /// + /// Check how many bytes would be read (up to `maxbytes`) at this pointer if + /// there is a bitmap, returns zero if there is no valid bitmap. + /// + @inlinable @inline(__always) + public static func portableDeserializeSize(buffer: [Int8], maxbytes: size_t) -> size_t { + return croaring.roaring64_bitmap_portable_deserialize_size(buffer, maxbytes) + } + + /// + /// How many bytes `frozenSerialize()` needs. Call `shrink()` first. + /// + @inlinable @inline(__always) + public func frozenSizeInBytes() -> size_t { + return croaring.roaring64_bitmap_frozen_size_in_bytes(self.ptr) + } + + /// + /// Serializes the bitmap in the "frozen" format, which mirrors the in-memory + /// layout so that `frozenView(bytes:)` can read it back without parsing. + /// `shrink()` must be called first. + /// + /// The format is neither portable across architectures nor stable across + /// releases of CRoaring; use `portableSerialize()` for storage. + /// + public func frozenSerialize() -> [UInt8] { + var buffer = [UInt8](repeating: 0, count: self.frozenSizeInBytes()) + guard !buffer.isEmpty else { return buffer } + buffer.withUnsafeMutableBytes { + _ = croaring.roaring64_bitmap_frozen_serialize( + self.ptr, + $0.baseAddress!.assumingMemoryBound(to: CChar.self) + ) + } + return buffer + } + + /// + /// Creates a read-only bitmap out of bytes produced by `frozenSerialize()`. + /// Returns nil if the bytes do not hold a valid frozen bitmap. + /// + /// The bytes are copied into a suitably aligned buffer owned by the returned + /// bitmap. The result must only be read, never modified. + /// + public static func frozenView(bytes: [UInt8]) -> Self? { + return withOwnedAlignedCopy(of: bytes) { buffer in + croaring.roaring64_bitmap_frozen_view( + buffer.assumingMemoryBound(to: CChar.self), + bytes.count + ) + } + } + + /// + /// Reads a bitmap serialized by `portableSerialize()` without copying the + /// container payloads out of the buffer, which is faster than + /// `portableDeserializeSafe(bytes:)` but yields a read-only bitmap. Returns + /// nil if the bytes do not hold a valid bitmap, and on big-endian machines, + /// where the payloads cannot be used where they sit. + /// + /// The bytes are copied into a buffer owned by the returned bitmap. The + /// result must only be read, never modified. Reading no more than + /// `bytes.count`, the call itself is safe, but a bitmap recovered from + /// arbitrary bytes may still be malformed: check `internalValidate()` before + /// using one that came from an untrusted source. + /// + public static func portableDeserializeFrozen(bytes: [UInt8]) -> Self? { + return withOwnedAlignedCopy(of: bytes) { buffer in + croaring.roaring64_bitmap_portable_deserialize_frozen( + buffer.assumingMemoryBound(to: CChar.self), + bytes.count + ) + } + } + + /// + /// Copies `bytes` into an aligned buffer, hands it to `makeBitmap`, and ties + /// the buffer's lifetime to the resulting bitmap. + /// + private static func withOwnedAlignedCopy( + of bytes: [UInt8], + _ makeBitmap: (UnsafeMutableRawPointer) -> OpaquePointer? + ) -> Self? { + let buffer = UnsafeMutableRawPointer.allocate( + byteCount: Swift.max(bytes.count, 1), + alignment: 64 + ) + bytes.withUnsafeBytes { source in + if let base = source.baseAddress { + buffer.copyMemory(from: base, byteCount: bytes.count) + } + } + guard let ptr = makeBitmap(buffer) else { + buffer.deallocate() + return nil + } + let bitmap = Self(ptr: ptr) + bitmap.frozenBacking = buffer + return bitmap + } + + /// + /// Encodable conformance, using the portable format. + /// + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(Data(self.portableSerialize()).base64EncodedString()) + } + + /// + /// Decodable conformance, using the portable format. + /// + public convenience init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let encoded = try container.decode(String.self) + guard let data = Data(base64Encoded: encoded) else { + throw DecodingError.dataCorruptedError( + in: container, + debugDescription: "not valid base64" + ) + } + let ptr: OpaquePointer? = data.withUnsafeBytes { + croaring.roaring64_bitmap_portable_deserialize_safe( + $0.baseAddress?.assumingMemoryBound(to: CChar.self), + data.count + ) + } + guard let ptr = ptr else { + throw DecodingError.dataCorruptedError( + in: container, + debugDescription: "not a valid serialized Roaring64Bitmap" + ) + } + self.init(ptr: ptr) + } +} diff --git a/Sources/SwiftRoaring/RoaringBitmap.swift b/Sources/SwiftRoaring/RoaringBitmap.swift index bdc9e4c..189d20d 100644 --- a/Sources/SwiftRoaring/RoaringBitmap.swift +++ b/Sources/SwiftRoaring/RoaringBitmap.swift @@ -9,12 +9,30 @@ public typealias RoaringStatistics = roaring_statistics_t /// /// Swift wrapper for CRoaring (a C/C++ implementation at https://github.com/RoaringBitmap/CRoaring) /// +/// Every function of CRoaring's public 32-bit API is reachable from this type, +/// except for five that have no Swift meaning: +/// `roaring_bitmap_init_with_capacity` and `roaring_bitmap_init_cleared` +/// initialize a `roaring_bitmap_t` the caller allocated, which `init()` and +/// `init(capacity:)` do here; `roaring_iterator_create`, +/// `roaring_uint32_iterator_copy` and `roaring_uint32_iterator_free` are the +/// heap-allocated iterator, replaced by the allocation-free +/// `RoaringBitmapIterator` value type. The variadic `roaring_bitmap_from` macro +/// corresponds to `init(values:)` and to array literals. +/// public final class RoaringBitmap: Sequence, Equatable, CustomStringConvertible, Hashable, ExpressibleByArrayLiteral, SetAlgebra, Codable { @usableFromInline var ptr: UnsafeMutablePointer + /// + /// When this bitmap is a *frozen view* created by `frozenView(bytes:)`, this + /// holds the aligned buffer the view reads from. It is deallocated, after the + /// bitmap itself, by `deinit`. It is `nil` for every ordinary bitmap. + /// + @usableFromInline + var frozenBacking: UnsafeMutableRawPointer? + public typealias Element = UInt32 ///////////////////////////////////////////////////////////////////////////// @@ -73,6 +91,7 @@ public final class RoaringBitmap: Sequence, Equatable, CustomStringConvertible, deinit { self.free() + self.frozenBacking?.deallocate() } ///////////////////////////////////////////////////////////////////////////// @@ -451,17 +470,17 @@ public final class RoaringBitmap: Sequence, Equatable, CustomStringConvertible, } /// - /// Copies a bitmap from src to dest. It is assumed that the pointer dest - /// is to an already allocated bitmap. The content of the dest bitmap is - /// freed/deleted. + /// Replaces the content of this bitmap with a copy of `other`. /// - /// It might be preferable and simpler to call `roaring_bitmap_copy` except - /// that `roaring_bitmap_overwrite` can save on memory allocations. + /// It might be preferable and simpler to call `copy()`, except that + /// `overwrite(with:)` can save on memory allocations. Returns false if the + /// copy could not be made. /// - /// - // func overwrite(dest: RoaringBitmap) -> Bool { - // return croaring.roaring_bitmap_overwrite(dest.ptr, self.ptr) - // } + @inlinable @inline(__always) + @discardableResult + public func overwrite(with other: RoaringBitmap) -> Bool { + return croaring.roaring_bitmap_overwrite(self.ptr, other.ptr) + } /// /// Add value x @@ -538,7 +557,7 @@ public final class RoaringBitmap: Sequence, Equatable, CustomStringConvertible, } /// - /// Remove value x + /// Remove value x, and return it if it was there. /// @inlinable @inline(__always) @discardableResult @@ -547,6 +566,15 @@ public final class RoaringBitmap: Sequence, Equatable, CustomStringConvertible, return value } + /// + /// Remove value x, without reporting whether it was there. Slightly faster + /// than `remove(_:)`. + /// + @inlinable @inline(__always) + public func discard(_ value: UInt32) { + croaring.roaring_bitmap_remove(self.ptr, value) + } + /// /// Remove all values in range [min, max] /// @@ -563,11 +591,13 @@ public final class RoaringBitmap: Sequence, Equatable, CustomStringConvertible, croaring.roaring_bitmap_remove_range(self.ptr, min, max) } - // /** Remove multiple values */ - // func removeMany(n_args: size_t, vals: [UInt32]) { - // let ptr: UnsafeMutablePointer = UnsafeMutablePointer(mutating: vals) - // croaring.roaring_bitmap_remove_many(self.ptr, n_args, ptr) - // } + /// + /// Remove multiple values, faster than repeatedly calling `remove(_:)`. + /// + @inlinable @inline(__always) + public func removeMany(values: [UInt32]) { + croaring.roaring_bitmap_remove_many(self.ptr, values.count, values) + } /// /// Remove value x @@ -675,20 +705,20 @@ public final class RoaringBitmap: Sequence, Equatable, CustomStringConvertible, return array } - // /** - // * Convert the bitmap to an array from "offset" by "limit". Write the output to "ans". - // * so, you can get data in paging. - // * caller is responsible to ensure that there is enough memory - // * allocated - // * (e.g., ans = malloc(roaring_bitmap_get_cardinality(limit) - // * * sizeof(uint32_t)) - // * Return false in case of failure (e.g., insufficient memory) - // */ - // public func toArrayRange(offset: size_t, limit: size_t) -> [UInt32] { - // let array: [UInt32] = [] - // _ = croaring.roaring_bitmap_range_uint32_array(self.ptr, offset, limit, array) - // return array - // } + /// + /// Convert a slice of the bitmap to an array: at most `limit` values, starting + /// at the `offset`-th smallest one. Useful to page through a large bitmap. + /// Returns nil in case of failure. + /// + public func toArray(offset: size_t, limit: size_t) -> [UInt32]? { + let cardinality = size_t(self.count) + let count = offset >= cardinality ? 0 : Swift.min(limit, cardinality - offset) + var array = [UInt32](repeating: 0, count: count) + guard croaring.roaring_bitmap_range_uint32_array(self.ptr, offset, limit, &array) else { + return nil + } + return array + } /// /// Remove run-length encoding even when it is more space efficient @@ -850,11 +880,18 @@ public final class RoaringBitmap: Sequence, Equatable, CustomStringConvertible, } /// - /// If the size of the roaring bitmap is strictly greater than rank, then this - /// function returns true and set the value to the the given rank. - /// Otherwise, it returns false. + /// Returns the element having the designated rank, where the smallest element + /// has rank 0. Returns nil if the bitmap holds `rank` elements or fewer. /// @inlinable @inline(__always) + public func select(rank: UInt32) -> UInt32? { + var value: UInt32 = 0 + guard croaring.roaring_bitmap_select(self.ptr, rank, &value) else { return nil } + return value + } + + @available(*, deprecated, message: "the result was discarded; use select(rank:) -> UInt32? instead") + @inlinable @inline(__always) public func select(rank: UInt32, value: UInt32) -> Bool { var cpy = value return croaring.roaring_bitmap_select(self.ptr, rank, &cpy) @@ -924,7 +961,7 @@ public final class RoaringBitmap: Sequence, Equatable, CustomStringConvertible, } /// - /// Creates a RoaringBitmapIterator. + /// Creates a RoaringBitmapIterator positioned on the smallest value. /// @inlinable @inline(__always) public func makeIterator() -> RoaringBitmapIterator { @@ -932,7 +969,20 @@ public final class RoaringBitmap: Sequence, Equatable, CustomStringConvertible, } /// - /// Structure used to iterate through values in a roaring bitmap + /// Creates a RoaringBitmapIterator positioned on the largest value, to be + /// walked backward with `previous()`. + /// + @inlinable @inline(__always) + public func makeLastIterator() -> RoaringBitmapIterator { + return RoaringBitmapIterator(bitmap: self, atLast: true) + } + + /// + /// Structure used to iterate through values in a roaring bitmap. + /// + /// The iterator is a value type holding the C iterator inline, so no + /// allocation takes place and copying it copies its position (the role + /// `roaring_uint32_iterator_copy` plays in C). /// @frozen public struct RoaringBitmapIterator: IteratorProtocol { @@ -949,6 +999,41 @@ public final class RoaringBitmap: Sequence, Equatable, CustomStringConvertible, self.bitmap = bitmap } + @inlinable @inline(__always) + init(bitmap: RoaringBitmap, atLast: Bool) { + self.i = roaring_uint32_iterator_t() + if atLast { + roaring_iterator_init_last(bitmap.ptr, &self.i) + } else { + roaring_iterator_init(bitmap.ptr, &self.i) + } + self.bitmap = bitmap + } + + /// + /// Whether the iterator currently points at a value. + /// + @inlinable @inline(__always) + public var hasValue: Bool { + return i.has_value + } + + /// + /// The value the iterator points at, undefined when `hasValue` is false. + /// + @inlinable @inline(__always) + public var currentValue: UInt32 { + return i.current_value + } + + /// + /// The value the iterator points at, or nil if it is exhausted. + /// + @inlinable @inline(__always) + public var value: UInt32? { + return i.has_value ? i.current_value : nil + } + @inlinable @inline(__always) public mutating func next() -> UInt32? { guard i.has_value else { return nil } @@ -956,6 +1041,122 @@ public final class RoaringBitmap: Sequence, Equatable, CustomStringConvertible, croaring.roaring_uint32_iterator_advance(&self.i) return val } + + /// + /// Moves the iterator to the next value. Returns true if it still points + /// at a value afterward. + /// + @inlinable @inline(__always) + @discardableResult + public mutating func advance() -> Bool { + return croaring.roaring_uint32_iterator_advance(&self.i) + } + + /// + /// Moves the iterator to the previous value. Returns true if it still + /// points at a value afterward. + /// + @inlinable @inline(__always) + @discardableResult + public mutating func previous() -> Bool { + return croaring.roaring_uint32_iterator_previous(&self.i) + } + + /// + /// Returns the previous value, walking the bitmap backward, or nil once + /// the iterator is exhausted. Pair with `makeLastIterator()`. + /// + @inlinable @inline(__always) + public mutating func nextBackward() -> UInt32? { + guard i.has_value else { return nil } + let val = i.current_value + croaring.roaring_uint32_iterator_previous(&self.i) + return val + } + + /// + /// Moves the iterator to the smallest value that is greater than or equal + /// to `value`. Returns true if such a value exists. + /// + @inlinable @inline(__always) + @discardableResult + public mutating func moveEqualOrLarger(_ value: UInt32) -> Bool { + return croaring.roaring_uint32_iterator_move_equalorlarger(&self.i, value) + } + + /// + /// Reads up to `count` values into an array, advancing the iterator past + /// them. The array is shorter than `count` when the bitmap runs out. + /// + public mutating func read(count: UInt32) -> [UInt32] { + var buffer = [UInt32](repeating: 0, count: Int(count)) + let read = buffer.withUnsafeMutableBufferPointer { buf in + croaring.roaring_uint32_iterator_read(&self.i, buf.baseAddress, count) + } + buffer.removeLast(Int(count) - Int(read)) + return buffer + } + + /// + /// Reads up to `count` values in decreasing order into an array, moving + /// the iterator backward past them. Pair with `makeLastIterator()`. + /// + public mutating func readBackward(count: UInt32) -> [UInt32] { + var buffer = [UInt32](repeating: 0, count: Int(count)) + let read = buffer.withUnsafeMutableBufferPointer { buf in + croaring.roaring_uint32_iterator_read_backward(&self.i, buf.baseAddress, count) + } + buffer.removeLast(Int(count) - Int(read)) + return buffer + } + + /// + /// Reads up to `count` maximal runs of consecutive values, advancing the + /// iterator past them. `{1, 2, 3, 5, 6}` reads as `[1...3, 5...6]`. + /// + public mutating func readRanges(count: Int) -> [ClosedRange] { + var buffer = [roaring_uint32_range_closed_t]( + repeating: roaring_uint32_range_closed_t(), count: count + ) + let read = buffer.withUnsafeMutableBufferPointer { buf in + croaring.roaring_uint32_iterator_read_ranges(&self.i, buf.baseAddress, count) + } + return buffer.prefix(read).map { $0.min...$0.max } + } + + /// + /// Reads up to `count` maximal runs of consecutive values in decreasing + /// order, moving the iterator backward past them. + /// + public mutating func readPreviousRanges(count: Int) -> [ClosedRange] { + var buffer = [roaring_uint32_range_closed_t]( + repeating: roaring_uint32_range_closed_t(), count: count + ) + let read = buffer.withUnsafeMutableBufferPointer { buf in + croaring.roaring_uint32_iterator_read_prev_ranges(&self.i, buf.baseAddress, count) + } + return buffer.prefix(read).map { $0.min...$0.max } + } + + /// + /// Advances the iterator past `count` values without reading them. + /// Returns how many were actually skipped. + /// + @inlinable @inline(__always) + @discardableResult + public mutating func skip(_ count: UInt32) -> UInt32 { + return croaring.roaring_uint32_iterator_skip(&self.i, count) + } + + /// + /// Moves the iterator backward past `count` values without reading them. + /// Returns how many were actually skipped. + /// + @inlinable @inline(__always) + @discardableResult + public mutating func skipBackward(_ count: UInt32) -> UInt32 { + return croaring.roaring_uint32_iterator_skip_backward(&self.i, count) + } } /// @@ -995,3 +1196,456 @@ extension RangeReplaceableCollection { self.reserveCapacity(capacity) } } + +///////////////////////////////////////////////////////////////////////////// +/// BULK (CONTEXT) OPERATIONS /// +///////////////////////////////////////////////////////////////////////////// + +extension RoaringBitmap { + /// + /// A bit of context usable with the `*Bulk` operations. + /// + /// Reusing a context across calls whose values are in increasing order lets + /// CRoaring skip the container lookup. A context may only be used with a + /// single bitmap, and any modification of that bitmap that does not go + /// through a `*Bulk` call taking the same context invalidates it. + /// + @frozen + public struct BulkContext { + @usableFromInline + internal var context: roaring_bulk_context_t + + @inlinable @inline(__always) + public init() { + self.context = roaring_bulk_context_t() + } + + /// + /// Forgets the cached container, making the context usable again after + /// the bitmap was modified out of band (or with another bitmap). + /// + @inlinable @inline(__always) + public mutating func reset() { + self.context = roaring_bulk_context_t() + } + } + + /// + /// Add value x, using context from a previous insert for speed optimization. + /// + @inlinable @inline(__always) + public func addBulk(_ value: UInt32, context: inout BulkContext) { + croaring.roaring_bitmap_add_bulk(self.ptr, &context.context, value) + } + + /// + /// Check if value x is present, using context from a previous lookup for + /// speed optimization. + /// + @inlinable @inline(__always) + public func containsBulk(_ value: UInt32, context: inout BulkContext) -> Bool { + return croaring.roaring_bitmap_contains_bulk(self.ptr, &context.context, value) + } +} + +///////////////////////////////////////////////////////////////////////////// +/// RANGES AND OFFSETS /// +///////////////////////////////////////////////////////////////////////////// + +extension RoaringBitmap { + /// + /// Remove all values in range [min, max] + /// + @inlinable @inline(__always) + public func remove(_ range: ClosedRange) { + croaring.roaring_bitmap_remove_range_closed( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// Remove all values in range [min, max) + /// + @inlinable @inline(__always) + public func remove(_ range: Range) { + croaring.roaring_bitmap_remove_range( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// Check whether all the values from `min` to `max` (both included) are present. + /// + @inlinable @inline(__always) + public func containsRangeClosed(min: UInt32, max: UInt32) -> Bool { + return croaring.roaring_bitmap_contains_range_closed(self.ptr, min, max) + } + + @inlinable @inline(__always) + public func contains(_ range: ClosedRange) -> Bool { + return croaring.roaring_bitmap_contains_range_closed( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// Returns the number of elements in range [min, max]. + /// + @inlinable @inline(__always) + public func rangeCardinalityClosed(min: UInt32, max: UInt32) -> UInt64 { + return croaring.roaring_bitmap_range_cardinality_closed(self.ptr, min, max) + } + + @inlinable @inline(__always) + public func cardinality(in range: Range) -> UInt64 { + return croaring.roaring_bitmap_range_cardinality( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + @inlinable @inline(__always) + public func cardinality(in range: ClosedRange) -> UInt64 { + return croaring.roaring_bitmap_range_cardinality_closed( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// Check whether the bitmap has any value in the range [x, y). + /// + @inlinable @inline(__always) + public func intersectWithRange(min: UInt64, max: UInt64) -> Bool { + return croaring.roaring_bitmap_intersect_with_range(self.ptr, min, max) + } + + @inlinable @inline(__always) + public func intersects(_ range: Range) -> Bool { + return croaring.roaring_bitmap_intersect_with_range( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// compute the negation of the roaring bitmap within a specified closed + /// interval: [rangeStart, rangeEnd]. + /// Areas outside the range are passed through unchanged. + /// + @inlinable @inline(__always) + public func flipClosed(rangeStart: UInt32, rangeEnd: UInt32) -> Self { + return Self(ptr: croaring.roaring_bitmap_flip_closed(self.ptr, rangeStart, rangeEnd)) + } + + @inlinable @inline(__always) + public func flip(_ range: ClosedRange) -> Self { + return Self(ptr: croaring.roaring_bitmap_flip_closed( + self.ptr, + range.lowerBound, + range.upperBound + )) + } + + /// + /// compute (in place) the negation of the roaring bitmap within a specified + /// closed interval: [rangeStart, rangeEnd]. + /// Areas outside the range are passed through unchanged. + /// + @inlinable @inline(__always) + public func flipInplaceClosed(rangeStart: UInt32, rangeEnd: UInt32) { + croaring.roaring_bitmap_flip_inplace_closed(self.ptr, rangeStart, rangeEnd) + } + + @inlinable @inline(__always) + public func flipInPlace(_ range: ClosedRange) { + croaring.roaring_bitmap_flip_inplace_closed( + self.ptr, + range.lowerBound, + range.upperBound + ) + } + + /// + /// Returns a new bitmap containing every value of this one shifted by + /// `offset`. Values that would fall outside `UInt32` are dropped. + /// + @inlinable @inline(__always) + public func addOffset(_ offset: Int64) -> Self { + return Self(ptr: croaring.roaring_bitmap_add_offset(self.ptr, offset)) + } +} + +///////////////////////////////////////////////////////////////////////////// +/// RANK/SELECT /// +///////////////////////////////////////////////////////////////////////////// + +extension RoaringBitmap { + /// + /// Returns the number of integers that are smaller or equal to each of + /// `values`, in one pass. `values` must be sorted in ascending order. + /// + public func rankMany(_ values: [UInt32]) -> [UInt64] { + var answers = [UInt64](repeating: 0, count: values.count) + guard !values.isEmpty else { return answers } + values.withUnsafeBufferPointer { input in + answers.withUnsafeMutableBufferPointer { output in + croaring.roaring_bitmap_rank_many( + self.ptr, + input.baseAddress, + input.baseAddress! + input.count, + output.baseAddress + ) + } + } + return answers + } + + /// + /// Returns the index of `value` in the sorted set, or nil when `value` is + /// not present. It is equivalent to, but faster than, `rank(value:) - 1`. + /// + @inlinable @inline(__always) + public func index(of value: UInt32) -> UInt64? { + let index = croaring.roaring_bitmap_get_index(self.ptr, value) + return index < 0 ? nil : UInt64(index) + } +} + +///////////////////////////////////////////////////////////////////////////// +/// COPY-ON-WRITE /// +///////////////////////////////////////////////////////////////////////////// + +extension RoaringBitmap { + /// + /// Whether copy-on-write is enabled. With copy-on-write, `copy()` and the + /// binary operations may share containers instead of duplicating them, + /// which is faster and more compact but makes the bitmaps unsafe to use + /// from several threads at once. + /// + /// Turning it off unshares every container that is currently shared. + /// + @inlinable @inline(__always) + public var copyOnWrite: Bool { + get { return croaring.roaring_bitmap_get_copy_on_write(self.ptr) } + set { croaring.roaring_bitmap_set_copy_on_write(self.ptr, newValue) } + } + + /// + /// Whether the bitmap holds at least one container shared with another + /// bitmap, which can only happen when copy-on-write is in use. + /// + @inlinable @inline(__always) + public var containsShared: Bool { + return croaring.roaring_contains_shared(self.ptr) + } + + /// + /// Gives the bitmap its own copy of every shared container. Returns false + /// if the allocation failed, in which case the bitmap is unchanged. + /// + @inlinable @inline(__always) + @discardableResult + public func unshareAll() -> Bool { + return croaring.roaring_unshare_all(self.ptr) + } +} + +///////////////////////////////////////////////////////////////////////////// +/// CLOSURE ITERATION /// +///////////////////////////////////////////////////////////////////////////// + +extension RoaringBitmap { + /// + /// Calls `body` once for every value of the bitmap, in increasing order, + /// stopping early as soon as `body` returns false. Returns true if the + /// whole bitmap was visited. + /// + /// This is faster than the `Sequence` conformance, at the cost of not being + /// able to interleave the iteration with other work. + /// + @discardableResult + public func iterate(_ body: (UInt32) -> Bool) -> Bool { + return withoutActuallyEscaping(body) { body in + var body = body + return withUnsafeMutablePointer(to: &body) { context in + croaring.roaring_iterate(self.ptr, { value, context in + let body = context!.assumingMemoryBound(to: ((UInt32) -> Bool).self) + return body.pointee(value) + }, UnsafeMutableRawPointer(context)) + } + } + } + + /// + /// Same as `iterate(_:)`, but each value is widened to 64 bits and combined + /// with `highBits`, which is useful when the bitmap stands for one slice of + /// a larger 64-bit set. + /// + @discardableResult + public func iterate64(highBits: UInt64, _ body: (UInt64) -> Bool) -> Bool { + return withoutActuallyEscaping(body) { body in + var body = body + return withUnsafeMutablePointer(to: &body) { context in + croaring.roaring_iterate64(self.ptr, { value, context in + let body = context!.assumingMemoryBound(to: ((UInt64) -> Bool).self) + return body.pointee(value) + }, highBits, UnsafeMutableRawPointer(context)) + } + } + } +} + +///////////////////////////////////////////////////////////////////////////// +/// CONVERSIONS /// +///////////////////////////////////////////////////////////////////////////// + +extension RoaringBitmap { + /// + /// Converts the bitmap to a plain uncompressed bitset, returned as its + /// 64-bit words: value `v` is present when bit `v % 64` of word `v / 64` is + /// set. Returns nil if the allocation failed. + /// + /// Beware that this needs about `maximum() / 8` bytes. + /// + public func toBitsetWords() -> [UInt64]? { + guard let bitset = croaring.bitset_create() else { return nil } + defer { croaring.bitset_free(bitset) } + guard croaring.roaring_bitmap_to_bitset(self.ptr, bitset) else { return nil } + guard let words = bitset.pointee.array else { return [] } + return [UInt64](UnsafeBufferPointer(start: words, count: bitset.pointee.arraysize)) + } +} + +///////////////////////////////////////////////////////////////////////////// +/// SERIALIZATION (EXTRAS) /// +///////////////////////////////////////////////////////////////////////////// + +extension RoaringBitmap { + /// + /// Read a bitmap written by `serialize(buffer:)`, reading no more than + /// `maxbytes` bytes. Returns nil if the buffer does not hold a valid bitmap. + /// + public static func deserializeSafe(buffer: [Int8], maxbytes: size_t) -> Self? { + guard let ptr = croaring.roaring_bitmap_deserialize_safe(buffer, maxbytes) else { + return nil + } + return Self(ptr: ptr) + } + + /// + /// How many bytes `frozenSerialize()` needs. + /// + @inlinable @inline(__always) + public func frozenSizeInBytes() -> size_t { + return croaring.roaring_bitmap_frozen_size_in_bytes(self.ptr) + } + + /// + /// Serializes the bitmap in the "frozen" format, which mirrors the in-memory + /// layout so that `frozenView(bytes:)` can read it back without parsing. + /// + /// The format is neither portable across architectures nor stable across + /// releases of CRoaring; use `portableSerialize(buffer:)` for storage. + /// + public func frozenSerialize() -> [UInt8] { + var buffer = [UInt8](repeating: 0, count: self.frozenSizeInBytes()) + guard !buffer.isEmpty else { return buffer } + buffer.withUnsafeMutableBytes { + croaring.roaring_bitmap_frozen_serialize( + self.ptr, + $0.baseAddress!.assumingMemoryBound(to: CChar.self) + ) + } + return buffer + } + + /// + /// Creates a read-only bitmap out of bytes produced by `frozenSerialize()`. + /// Returns nil if the bytes do not hold a valid frozen bitmap. + /// + /// The bytes are copied into a suitably aligned buffer owned by the returned + /// bitmap. The result must only be read, never modified. + /// + public static func frozenView(bytes: [UInt8]) -> Self? { + return withOwnedAlignedCopy(of: bytes, alignment: 32) { buffer in + croaring.roaring_bitmap_frozen_view( + buffer.assumingMemoryBound(to: CChar.self), + bytes.count + ).map { UnsafeMutablePointer(mutating: $0) } + } + } + + /// + /// Reads a bitmap serialized by `portableSerialize(buffer:)` without copying + /// the containers out of the buffer, which is faster than + /// `portableDeserialize(buffer:)` but yields a read-only bitmap. + /// + /// The bytes are copied into a buffer owned by the returned bitmap. This is + /// unsafe in the same ways the underlying C function is: it trusts the bytes + /// to hold a valid serialized bitmap. + /// + public static func portableDeserializeFrozen(bytes: [UInt8]) -> Self? { + return withOwnedAlignedCopy(of: bytes, alignment: 32) { buffer in + croaring.roaring_bitmap_portable_deserialize_frozen( + buffer.assumingMemoryBound(to: CChar.self) + ) + } + } + + /// + /// Copies `bytes` into an aligned buffer, hands it to `makeBitmap`, and ties + /// the buffer's lifetime to the resulting bitmap. + /// + private static func withOwnedAlignedCopy( + of bytes: [UInt8], + alignment: Int, + _ makeBitmap: (UnsafeMutableRawPointer) -> UnsafeMutablePointer? + ) -> Self? { + let buffer = UnsafeMutableRawPointer.allocate( + byteCount: Swift.max(bytes.count, 1), + alignment: alignment + ) + bytes.withUnsafeBytes { source in + if let base = source.baseAddress { + buffer.copyMemory(from: base, byteCount: bytes.count) + } + } + guard let ptr = makeBitmap(buffer) else { + buffer.deallocate() + return nil + } + let bitmap = Self(ptr: ptr) + bitmap.frozenBacking = buffer + return bitmap + } +} + +///////////////////////////////////////////////////////////////////////////// +/// VALIDATION /// +///////////////////////////////////////////////////////////////////////////// + +extension RoaringBitmap { + /// + /// (For advanced users.) + /// Checks the internal invariants of the bitmap, which is mostly useful on + /// bitmaps that came from an untrusted source. Returns nil when the bitmap + /// is well formed, and the reason it is not otherwise. + /// + public func internalValidate() -> String? { + var reason: UnsafePointer? + guard !croaring.roaring_bitmap_internal_validate(self.ptr, &reason) else { + return nil + } + return reason.map { String(cString: $0) } ?? "unknown reason" + } +} diff --git a/Tests/swiftRoaringTests/roaring32ExtendedTests.swift b/Tests/swiftRoaringTests/roaring32ExtendedTests.swift new file mode 100644 index 0000000..f43796d --- /dev/null +++ b/Tests/swiftRoaringTests/roaring32ExtendedTests.swift @@ -0,0 +1,316 @@ +import XCTest +@testable import SwiftRoaring + +/// +/// Covers the 32-bit C functions that the original wrapper left out. +/// +class roaring32ExtendedTests: XCTestCase { + + func testBulkContext() { + let a = RoaringBitmap() + var context = RoaringBitmap.BulkContext() + for value in stride(from: UInt32(0), to: 1000, by: 3) { + a.addBulk(value, context: &context) + } + XCTAssertEqual(a.toArray(), Array(stride(from: UInt32(0), to: 1000, by: 3))) + + context.reset() + for value in stride(from: UInt32(0), to: 1000, by: 3) { + XCTAssertTrue(a.containsBulk(value, context: &context)) + } + context.reset() + XCTAssertFalse(a.containsBulk(1, context: &context)) + } + + func testOverwrite() { + let a: RoaringBitmap = [1, 2, 3] + let b: RoaringBitmap = [99] + XCTAssertTrue(b.overwrite(with: a)) + XCTAssertEqual(b.toArray(), [1, 2, 3]) + // The source is untouched, and the two are independent. + b.add(4) + XCTAssertEqual(a.toArray(), [1, 2, 3]) + } + + func testDiscard() { + let a: RoaringBitmap = [1, 2, 3] + a.discard(2) + a.discard(99) + XCTAssertEqual(a.toArray(), [1, 3]) + } + + func testRemoveMany() { + let a = RoaringBitmap(range: 0..<10, step: 1) + a.removeMany(values: [0, 2, 4, 100]) + XCTAssertEqual(a.toArray(), [1, 3, 5, 6, 7, 8, 9]) + } + + func testRemoveRangeSugar() { + let a = RoaringBitmap(range: 0..<10, step: 1) + a.remove(2...4) + XCTAssertEqual(a.toArray(), [0, 1, 5, 6, 7, 8, 9]) + a.remove(5..<8) + XCTAssertEqual(a.toArray(), [0, 1, 8, 9]) + } + + func testContainsAndCardinalityRanges() { + let a = RoaringBitmap(range: 10..<21, step: 1) + XCTAssertTrue(a.contains(10...20)) + XCTAssertTrue(a.containsRangeClosed(min: 10, max: 20)) + XCTAssertFalse(a.contains(10...21)) + + XCTAssertEqual(a.cardinality(in: 12...14), 3) + XCTAssertEqual(a.cardinality(in: 12..<14), 2) + XCTAssertEqual(a.rangeCardinalityClosed(min: 0, max: 15), 6) + XCTAssertEqual(a.rangeCardinality(min: 0, max: 15), 5) + } + + func testIntersectWithRange() { + let a: RoaringBitmap = [10, 20, 30] + XCTAssertTrue(a.intersectWithRange(min: 0, max: 11)) + XCTAssertFalse(a.intersectWithRange(min: 0, max: 10)) + XCTAssertTrue(a.intersects(20..<21)) + XCTAssertFalse(a.intersects(21..<30)) + } + + func testFlipClosed() { + let a: RoaringBitmap = [1, 3] + XCTAssertEqual(a.flipClosed(rangeStart: 0, rangeEnd: 3).toArray(), [0, 2]) + XCTAssertEqual(a.flip(0...3).toArray(), [0, 2]) + XCTAssertEqual(a.toArray(), [1, 3]) + + let b: RoaringBitmap = [1, 3] + b.flipInplaceClosed(rangeStart: 0, rangeEnd: 3) + XCTAssertEqual(b.toArray(), [0, 2]) + b.flipInPlace(0...3) + XCTAssertEqual(b.toArray(), [1, 3]) + } + + func testAddOffset() { + let a: RoaringBitmap = [1, 2, 3] + XCTAssertEqual(a.addOffset(10).toArray(), [11, 12, 13]) + XCTAssertEqual(a.addOffset(-1).toArray(), [0, 1, 2]) + // Values that would fall outside UInt32 are dropped. + XCTAssertEqual(a.addOffset(-2).toArray(), [0, 1]) + } + + func testSelect() { + let a: RoaringBitmap = [10, 20, 30] + XCTAssertEqual(a.select(rank: 0), 10) + XCTAssertEqual(a.select(rank: 2), 30) + XCTAssertNil(a.select(rank: 3)) + } + + func testRankMany() { + let a: RoaringBitmap = [1, 5, 9] + XCTAssertEqual(a.rankMany([0, 1, 5, 6, 9, 100]), [0, 1, 2, 2, 3, 3]) + XCTAssertEqual(a.rankMany([]), []) + } + + func testIndexOf() { + let a: RoaringBitmap = [10, 20, 30] + XCTAssertEqual(a.index(of: 10), 0) + XCTAssertEqual(a.index(of: 30), 2) + XCTAssertNil(a.index(of: 11)) + } + + func testCopyOnWrite() { + let a = RoaringBitmap(range: 0..<1000, step: 1) + XCTAssertFalse(a.copyOnWrite) + XCTAssertFalse(a.containsShared) + + a.copyOnWrite = true + XCTAssertTrue(a.copyOnWrite) + + let b = a.copy() + XCTAssertTrue(a.containsShared) + XCTAssertTrue(b.containsShared) + XCTAssertEqual(a, b) + + XCTAssertTrue(b.unshareAll()) + XCTAssertFalse(b.containsShared) + + a.copyOnWrite = false + XCTAssertFalse(a.copyOnWrite) + XCTAssertFalse(a.containsShared) + XCTAssertEqual(a, b) + } + + func testIterate() { + let a = RoaringBitmap(range: 0..<100, step: 7) + var seen: [UInt32] = [] + XCTAssertTrue(a.iterate { seen.append($0); return true }) + XCTAssertEqual(seen, Array(stride(from: UInt32(0), to: 100, by: 7))) + + var partial: [UInt32] = [] + XCTAssertFalse(a.iterate { partial.append($0); return partial.count < 3 }) + XCTAssertEqual(partial, [0, 7, 14]) + } + + func testIterate64() { + let a: RoaringBitmap = [1, 2, 3] + var seen: [UInt64] = [] + XCTAssertTrue(a.iterate64(highBits: 1 << 32) { seen.append($0); return true }) + XCTAssertEqual(seen, [(1 << 32) + 1, (1 << 32) + 2, (1 << 32) + 3]) + } + + func testIteratorNavigation() { + let a: RoaringBitmap = [1, 2, 3, 5, 6, 100] + + var forward = a.makeIterator() + XCTAssertTrue(forward.hasValue) + XCTAssertEqual(forward.currentValue, 1) + XCTAssertEqual(forward.value, 1) + XCTAssertTrue(forward.advance()) + XCTAssertEqual(forward.value, 2) + + // The iterator is a value type: the copy moves on its own. + var fork = forward + XCTAssertTrue(fork.advance()) + XCTAssertEqual(fork.value, 3) + XCTAssertEqual(forward.value, 2) + + XCTAssertTrue(forward.previous()) + XCTAssertEqual(forward.value, 1) + XCTAssertFalse(forward.previous()) + XCTAssertNil(forward.value) + + var seek = a.makeIterator() + XCTAssertTrue(seek.moveEqualOrLarger(4)) + XCTAssertEqual(seek.value, 5) + XCTAssertFalse(seek.moveEqualOrLarger(101)) + + var backward = a.makeLastIterator() + var reversed: [UInt32] = [] + while let value = backward.nextBackward() { + reversed.append(value) + } + XCTAssertEqual(reversed, [100, 6, 5, 3, 2, 1]) + } + + func testIteratorRead() { + let a = RoaringBitmap(range: 0..<10, step: 1) + var it = a.makeIterator() + XCTAssertEqual(it.read(count: 3), [0, 1, 2]) + XCTAssertEqual(it.read(count: 100), [3, 4, 5, 6, 7, 8, 9]) + XCTAssertEqual(it.read(count: 1), []) + + var back = a.makeLastIterator() + XCTAssertEqual(back.readBackward(count: 3), [9, 8, 7]) + } + + func testIteratorSkip() { + let a = RoaringBitmap(range: 0..<10, step: 1) + var it = a.makeIterator() + XCTAssertEqual(it.skip(4), 4) + XCTAssertEqual(it.value, 4) + XCTAssertEqual(it.skip(100), 6) + XCTAssertNil(it.value) + + var back = a.makeLastIterator() + XCTAssertEqual(back.skipBackward(4), 4) + XCTAssertEqual(back.value, 5) + } + + func testIteratorReadRanges() { + let a: RoaringBitmap = [1, 2, 3, 5, 6, 100] + var it = a.makeIterator() + XCTAssertEqual(it.readRanges(count: 2), [1...3, 5...6]) + XCTAssertEqual(it.readRanges(count: 10), [100...100]) + XCTAssertEqual(it.readRanges(count: 10), []) + + var back = a.makeLastIterator() + XCTAssertEqual(back.readPreviousRanges(count: 2), [100...100, 5...6]) + } + + func testToArrayPaged() { + let a = RoaringBitmap(range: 0..<100, step: 1) + XCTAssertEqual(a.toArray(offset: 0, limit: 3), [0, 1, 2]) + XCTAssertEqual(a.toArray(offset: 97, limit: 10), [97, 98, 99]) + XCTAssertEqual(a.toArray(offset: 100, limit: 10), []) + } + + func testToBitsetWords() { + let a: RoaringBitmap = [0, 1, 63, 64, 130] + guard let words = a.toBitsetWords() else { + return XCTFail("could not build the bitset") + } + XCTAssertGreaterThanOrEqual(words.count, 3) + for value in a { + let word = words[Int(value / 64)] + XCTAssertEqual((word >> (UInt64(value) % 64)) & 1, 1) + } + XCTAssertEqual(words.reduce(0) { $0 + $1.nonzeroBitCount }, 5) + // An empty bitmap has no bit set, whatever the number of words. + XCTAssertEqual(RoaringBitmap().toBitsetWords()?.filter { $0 != 0 }, []) + } + + func testDeserializeSafe() { + let a = RoaringBitmap(range: 0..<100, step: 3) + var buffer = [Int8](repeating: 0, count: a.sizeInBytes()) + XCTAssertEqual(a.serialize(buffer: &buffer), buffer.count) + + XCTAssertEqual( + RoaringBitmap.deserializeSafe(buffer: buffer, maxbytes: buffer.count), + a + ) + // A buffer too short for the header is rejected instead of over-reading. + XCTAssertNil(RoaringBitmap.deserializeSafe(buffer: buffer, maxbytes: 1)) + } + + func testFrozenSerialization() { + let a = RoaringBitmap(range: 0..<1000, step: 3) + _ = a.runOptimize() + _ = a.shrink() + + let bytes = a.frozenSerialize() + XCTAssertEqual(bytes.count, a.frozenSizeInBytes()) + + guard let view = RoaringBitmap.frozenView(bytes: bytes) else { + return XCTFail("frozen view could not be created") + } + XCTAssertEqual(view, a) + XCTAssertEqual(view.toArray(), a.toArray()) + } + + func testPortableDeserializeFrozen() { + let a = RoaringBitmap(range: 0..<1000, step: 3) + var buffer = [Int8](repeating: 0, count: a.portableSizeInBytes()) + XCTAssertEqual(a.portableSerialize(buffer: &buffer), buffer.count) + + let bytes = buffer.map { UInt8(bitPattern: $0) } + guard let view = RoaringBitmap.portableDeserializeFrozen(bytes: bytes) else { + return XCTFail("frozen bitmap could not be read") + } + XCTAssertEqual(view, a) + XCTAssertEqual(view.toArray(), a.toArray()) + } + + func testEmptyBitmapRoundTrips() { + let empty = RoaringBitmap() + _ = empty.shrink() + + guard let frozen = RoaringBitmap.frozenView(bytes: empty.frozenSerialize()) else { + return XCTFail("frozen view of an empty bitmap could not be created") + } + XCTAssertTrue(frozen.isEmpty) + + var buffer = [Int8](repeating: 0, count: empty.portableSizeInBytes()) + _ = empty.portableSerialize(buffer: &buffer) + let bytes = buffer.map { UInt8(bitPattern: $0) } + XCTAssertEqual(RoaringBitmap.portableDeserializeFrozen(bytes: bytes)?.isEmpty, true) + + XCTAssertEqual(empty.toArray(offset: 0, limit: 10), []) + XCTAssertEqual(empty.rankMany([1, 2]), [0, 0]) + XCTAssertNil(empty.select(rank: 0)) + XCTAssertNil(empty.index(of: 0)) + XCTAssertTrue(empty.iterate { _ in true }) + } + + func testInternalValidate() { + let a = RoaringBitmap(range: 0..<1000, step: 3) + XCTAssertNil(a.internalValidate()) + _ = a.runOptimize() + XCTAssertNil(a.internalValidate()) + } +} diff --git a/Tests/swiftRoaringTests/roaring64Tests.swift b/Tests/swiftRoaringTests/roaring64Tests.swift new file mode 100644 index 0000000..962eec2 --- /dev/null +++ b/Tests/swiftRoaringTests/roaring64Tests.swift @@ -0,0 +1,436 @@ +import XCTest +@testable import SwiftRoaring + +class roaring64Tests: XCTestCase { + var rbm: Roaring64Bitmap! + + /// A few values spread over several high-48-bit partitions. + let spread: [UInt64] = [ + 0, + 1, + 1 << 16, + 1 << 32, + (1 << 32) + 1, + 1 << 48, + UInt64.max + ] + + override func setUp() { + super.setUp() + rbm = Roaring64Bitmap() + } + + func testAddContainsRemove() { + for value in spread { + rbm.add(value) + } + XCTAssertEqual(rbm.count, UInt64(spread.count)) + for value in spread { + XCTAssertTrue(rbm.contains(value)) + } + XCTAssertFalse(rbm.contains(2)) + XCTAssertEqual(rbm.remove(1 << 32), 1 << 32) + XCTAssertNil(rbm.remove(1 << 32)) + XCTAssertFalse(rbm.contains(1 << 32)) + } + + func testArrayLiteralAndValues() { + let a: Roaring64Bitmap = [3, 1, 2, 1] + XCTAssertEqual(a.toArray(), [1, 2, 3]) + XCTAssertEqual(Roaring64Bitmap(values: spread).toArray(), spread.sorted()) + } + + func testDiscard() { + let a: Roaring64Bitmap = [1, 2, 1 << 40] + a.discard(2) + a.discard(99) + XCTAssertEqual(a.toArray(), [1, 1 << 40]) + } + + func testAddManyRemoveMany() { + rbm.addMany(values: spread) + XCTAssertEqual(rbm.count, UInt64(spread.count)) + rbm.removeMany(values: [0, 1, UInt64.max]) + XCTAssertEqual(rbm.toArray(), [1 << 16, 1 << 32, (1 << 32) + 1, 1 << 48]) + } + + func testAddCheckedAndUpdate() { + XCTAssertTrue(rbm.addCheck(7)) + XCTAssertFalse(rbm.addCheck(7)) + XCTAssertEqual(rbm.update(with: 7), 7) + XCTAssertNil(rbm.update(with: 8)) + } + + func testRanges() { + rbm.add(10...20) + XCTAssertEqual(rbm.count, 11) + XCTAssertTrue(rbm.contains(10...20)) + XCTAssertTrue(rbm.contains(10..<21)) + XCTAssertFalse(rbm.contains(10...21)) + XCTAssertEqual(rbm.cardinality(in: 12...14), 3) + XCTAssertEqual(rbm.cardinality(in: 12..<14), 2) + XCTAssertEqual(rbm.rangeCardinality(min: 0, max: 15), 5) + XCTAssertEqual(rbm.rangeCardinalityClosed(min: 0, max: 15), 6) + + rbm.remove(12...14) + XCTAssertEqual(rbm.toArray(), [10, 11, 15, 16, 17, 18, 19, 20]) + rbm.remove(15..<17) + XCTAssertEqual(rbm.toArray(), [10, 11, 17, 18, 19, 20]) + + XCTAssertTrue(rbm.intersects(0..<11)) + XCTAssertFalse(rbm.intersects(0..<10)) + XCTAssertFalse(rbm.intersectWithRange(min: 12, max: 17)) + } + + func testInitRange() { + let a = Roaring64Bitmap(min: 0, max: 1000, step: 50) + XCTAssertEqual(a.toArray(), Array(stride(from: UInt64(0), to: 1000, by: 50))) + let b = Roaring64Bitmap(range: 0..<1000, step: 50) + XCTAssertEqual(a, b) + // An empty or degenerate range yields an empty bitmap rather than crashing. + XCTAssertTrue(Roaring64Bitmap(min: 10, max: 10, step: 1).isEmpty) + XCTAssertTrue(Roaring64Bitmap(min: 0, max: 10, step: 0).isEmpty) + } + + func testSetOperations() { + let a: Roaring64Bitmap = [1, 2, 3, 1 << 40] + let b: Roaring64Bitmap = [3, 4, 1 << 40] + + XCTAssertEqual((a & b).toArray(), [3, 1 << 40]) + XCTAssertEqual((a | b).toArray(), [1, 2, 3, 4, 1 << 40]) + XCTAssertEqual((a ^ b).toArray(), [1, 2, 4]) + XCTAssertEqual((a - b).toArray(), [1, 2]) + + XCTAssertEqual(a.intersectionCount(b), 2) + XCTAssertEqual(a.unionCount(b), 5) + XCTAssertEqual(a.symmetricDifferenceCount(b), 3) + XCTAssertEqual(a.subtractingCount(b), 2) + XCTAssertEqual(a.jaccardIndex(b), 2.0 / 5.0) + + XCTAssertTrue(a.intersect(b)) + XCTAssertFalse(a.isDisjoint(with: b)) + XCTAssertTrue(a.isDisjoint(with: [5, 6] as Roaring64Bitmap)) + + let sub: Roaring64Bitmap = [1, 2] + XCTAssertTrue(sub.isSubset(of: a)) + XCTAssertTrue(sub.isStrictSubset(of: a)) + XCTAssertFalse(a.isStrictSubset(of: a)) + XCTAssertTrue(a.isSubset(of: a)) + } + + func testInPlaceSetOperations() { + let a: Roaring64Bitmap = [1, 2, 3] + a &= [2, 3, 4] + XCTAssertEqual(a.toArray(), [2, 3]) + a |= [9] + XCTAssertEqual(a.toArray(), [2, 3, 9]) + a ^= [3, 10] + XCTAssertEqual(a.toArray(), [2, 9, 10]) + a -= [9] + XCTAssertEqual(a.toArray(), [2, 10]) + } + + func testFlip() { + let a: Roaring64Bitmap = [1, 3] + XCTAssertEqual(a.flip(0..<4).toArray(), [0, 2]) + XCTAssertEqual(a.flip(0...3).toArray(), [0, 2]) + XCTAssertEqual(a.flip(rangeStart: 0, rangeEnd: 4).toArray(), [0, 2]) + XCTAssertEqual(a.flipClosed(rangeStart: 0, rangeEnd: 3).toArray(), [0, 2]) + // The originals are untouched. + XCTAssertEqual(a.toArray(), [1, 3]) + + let b: Roaring64Bitmap = [1, 3] + b.flipInPlace(0..<4) + XCTAssertEqual(b.toArray(), [0, 2]) + b.flipInPlace(0...3) + XCTAssertEqual(b.toArray(), [1, 3]) + b.flipInplace(rangeStart: 0, rangeEnd: 4) + XCTAssertEqual(b.toArray(), [0, 2]) + b.flipInplaceClosed(rangeStart: 0, rangeEnd: 3) + XCTAssertEqual(b.toArray(), [1, 3]) + } + + func testOffsets() { + let a: Roaring64Bitmap = [1, 2, 1 << 40] + XCTAssertEqual(a.addOffset(10).toArray(), [11, 12, (1 << 40) + 10]) + XCTAssertEqual(a.subtractOffset(1).toArray(), [0, 1, (1 << 40) - 1]) + // Values that would underflow are dropped. + XCTAssertEqual(a.subtractOffset(2).toArray(), [0, (1 << 40) - 2]) + } + + func testCopyAndOverwrite() { + let a: Roaring64Bitmap = [1, 2, 1 << 40] + let b = a.copy() + XCTAssertEqual(a, b) + b.add(3) + XCTAssertNotEqual(a, b) + + let c: Roaring64Bitmap = [99] + c.overwrite(with: a) + XCTAssertEqual(c, a) + } + + func testMinMaxRankSelectIndex() { + let a: Roaring64Bitmap = [1, 5, 1 << 40] + XCTAssertEqual(a.min(), 1) + XCTAssertEqual(a.max(), 1 << 40) + XCTAssertEqual(a.first, 1) + XCTAssertEqual(a.last, 1 << 40) + + XCTAssertEqual(a.rank(value: 5), 2) + XCTAssertEqual(a.rank(value: 4), 1) + + XCTAssertEqual(a.select(rank: 0), 1) + XCTAssertEqual(a.select(rank: 2), 1 << 40) + XCTAssertNil(a.select(rank: 3)) + + XCTAssertEqual(a.index(of: 5), 1) + XCTAssertEqual(a.index(of: 1 << 40), 2) + XCTAssertNil(a.index(of: 6)) + + let empty = Roaring64Bitmap() + XCTAssertNil(empty.min()) + XCTAssertNil(empty.max()) + } + + func testPopFirstPopLast() { + let a: Roaring64Bitmap = [1, 2, 1 << 40] + XCTAssertEqual(a.popFirst(), 1) + XCTAssertEqual(a.popLast(), 1 << 40) + XCTAssertEqual(a.popFirst(), 2) + XCTAssertNil(a.popFirst()) + XCTAssertNil(a.popLast()) + XCTAssertTrue(a.isEmpty) + } + + func testClearAndRemoveAllWhere() { + rbm.addMany(values: spread) + rbm.removeAll { $0 < (1 << 32) } + XCTAssertEqual(rbm.toArray(), [1 << 32, (1 << 32) + 1, 1 << 48, UInt64.max]) + rbm.removeAll() + XCTAssertTrue(rbm.isEmpty) + } + + func testSequenceIteration() { + rbm.addMany(values: spread) + XCTAssertEqual(Array(rbm), spread.sorted()) + XCTAssertTrue(rbm.elementsEqual(spread.sorted())) + } + + func testIteratorNavigation() { + let a = Roaring64Bitmap(values: spread) + + let forward = a.makeIterator() + XCTAssertTrue(forward.hasValue) + XCTAssertEqual(forward.currentValue, 0) + XCTAssertEqual(forward.value, 0) + XCTAssertTrue(forward.advance()) + XCTAssertEqual(forward.value, 1) + + let fork = forward.copy() + XCTAssertTrue(fork.advance()) + XCTAssertEqual(fork.value, 1 << 16) + // The fork moved independently. + XCTAssertEqual(forward.value, 1) + + XCTAssertTrue(forward.previous()) + XCTAssertEqual(forward.value, 0) + XCTAssertFalse(forward.previous()) + XCTAssertNil(forward.value) + + forward.reinit() + XCTAssertEqual(forward.value, 0) + forward.reinitLast() + XCTAssertEqual(forward.value, UInt64.max) + + let backward = a.makeLastIterator() + var reversed: [UInt64] = [] + while let value = backward.nextBackward() { + reversed.append(value) + } + XCTAssertEqual(reversed, spread.sorted().reversed()) + + let seek = a.makeIterator() + XCTAssertTrue(seek.moveEqualOrLarger(1 << 32)) + XCTAssertEqual(seek.value, 1 << 32) + XCTAssertTrue(seek.moveEqualOrLarger(UInt64.max - 1)) + XCTAssertEqual(seek.value, UInt64.max) + XCTAssertFalse(seek.advance()) + } + + func testIteratorRead() { + let a = Roaring64Bitmap(values: spread) + let it = a.makeIterator() + XCTAssertEqual(it.read(count: 3), [0, 1, 1 << 16]) + // Asking for more than what is left yields a shorter array. + XCTAssertEqual(it.read(count: 100), [1 << 32, (1 << 32) + 1, 1 << 48, UInt64.max]) + XCTAssertEqual(it.read(count: 1), []) + + let back = a.makeLastIterator() + XCTAssertEqual(back.readBackward(count: 2), [UInt64.max, 1 << 48]) + } + + func testIteratorReadRanges() { + let a: Roaring64Bitmap = [1, 2, 3, 5, 6, 100] + let it = a.makeIterator() + XCTAssertEqual(it.readRanges(count: 2), [1...3, 5...6]) + XCTAssertEqual(it.readRanges(count: 10), [100...100]) + XCTAssertEqual(it.readRanges(count: 10), []) + + let back = a.makeLastIterator() + XCTAssertEqual(back.readPreviousRanges(count: 2), [100...100, 5...6]) + } + + func testClosureIteration() { + let a = Roaring64Bitmap(values: spread) + var seen: [UInt64] = [] + XCTAssertTrue(a.iterate { seen.append($0); return true }) + XCTAssertEqual(seen, spread.sorted()) + + var partial: [UInt64] = [] + XCTAssertFalse(a.iterate { partial.append($0); return partial.count < 3 }) + XCTAssertEqual(partial, Array(spread.sorted().prefix(3))) + } + + func testBulkContext() { + var context = Roaring64Bitmap.BulkContext() + for value in spread.sorted() { + rbm.addBulk(value, context: &context) + } + XCTAssertEqual(rbm.toArray(), spread.sorted()) + + context.reset() + for value in spread.sorted() { + XCTAssertTrue(rbm.containsBulk(value, context: &context)) + } + + context.reset() + rbm.removeBulk(1 << 32, context: &context) + XCTAssertFalse(rbm.contains(1 << 32)) + } + + func testOptimizationAndStatistics() { + rbm.add(0...10_000) + XCTAssertTrue(rbm.runOptimize()) + XCTAssertGreaterThan(rbm.statistics().n_run_containers, 0) + _ = rbm.shrink() + XCTAssertTrue(rbm.removeRunCompression()) + XCTAssertEqual(rbm.statistics().n_run_containers, 0) + XCTAssertEqual(rbm.statistics().cardinality, 10_001) + XCTAssertNil(rbm.internalValidate()) + } + + func testPortableSerialization() { + rbm.addMany(values: spread) + let bytes = rbm.portableSerialize() + XCTAssertEqual(bytes.count, rbm.portableSizeInBytes()) + + let back = Roaring64Bitmap.portableDeserializeSafe(bytes: bytes) + XCTAssertEqual(back, rbm) + + // Truncated input is rejected rather than crashing. + XCTAssertNil(Roaring64Bitmap.portableDeserializeSafe(bytes: Array(bytes.prefix(3)))) + + var buffer = [Int8](repeating: 0, count: rbm.portableSizeInBytes()) + XCTAssertEqual(rbm.portableSerialize(buffer: &buffer), buffer.count) + XCTAssertEqual( + Roaring64Bitmap.portableDeserializeSize(buffer: buffer, maxbytes: buffer.count), + buffer.count + ) + XCTAssertEqual( + Roaring64Bitmap.portableDeserializeSafe(buffer: buffer, maxbytes: buffer.count), + rbm + ) + } + + func testFrozenSerialization() { + rbm.addMany(values: spread) + rbm.add(1000...2000) + _ = rbm.shrink() + + let bytes = rbm.frozenSerialize() + XCTAssertEqual(bytes.count, rbm.frozenSizeInBytes()) + + guard let view = Roaring64Bitmap.frozenView(bytes: bytes) else { + return XCTFail("frozen view could not be created") + } + XCTAssertEqual(view, rbm) + XCTAssertEqual(view.toArray(), rbm.toArray()) + } + + func testPortableDeserializeFrozen() { + rbm.addMany(values: spread) + rbm.add(1_000...2_000) + _ = rbm.runOptimize() + + let bytes = rbm.portableSerialize() + guard let view = Roaring64Bitmap.portableDeserializeFrozen(bytes: bytes) else { + return XCTFail("frozen bitmap could not be read") + } + XCTAssertNil(view.internalValidate()) + XCTAssertEqual(view, rbm) + XCTAssertEqual(view.toArray(), rbm.toArray()) + + // Truncated input is rejected rather than over-read. + XCTAssertNil(Roaring64Bitmap.portableDeserializeFrozen(bytes: Array(bytes.prefix(3)))) + } + + func testEmptyBitmapRoundTrips() { + let empty = Roaring64Bitmap() + _ = empty.shrink() + + guard let frozen = Roaring64Bitmap.frozenView(bytes: empty.frozenSerialize()) else { + return XCTFail("frozen view of an empty bitmap could not be created") + } + XCTAssertTrue(frozen.isEmpty) + + XCTAssertEqual( + Roaring64Bitmap.portableDeserializeSafe(bytes: empty.portableSerialize())?.isEmpty, + true + ) + XCTAssertEqual( + Roaring64Bitmap.portableDeserializeFrozen(bytes: empty.portableSerialize())?.isEmpty, + true + ) + XCTAssertEqual(empty.toArray(), []) + XCTAssertNil(empty.select(rank: 0)) + XCTAssertNil(empty.index(of: 0)) + XCTAssertTrue(empty.iterate { _ in true }) + XCTAssertNil(empty.makeIterator().next()) + XCTAssertNil(empty.makeLastIterator().nextBackward()) + } + + func testCodable() throws { + rbm.addMany(values: spread) + let encoded = try JSONEncoder().encode(rbm) + let decoded = try JSONDecoder().decode(Roaring64Bitmap.self, from: encoded) + XCTAssertEqual(decoded, rbm) + + let garbage = try JSONEncoder().encode("not a bitmap") + XCTAssertThrowsError(try JSONDecoder().decode(Roaring64Bitmap.self, from: garbage)) + } + + func testFromRoaringBitmap() { + let source: RoaringBitmap = [1, 2, 3] + + let copied = Roaring64Bitmap(source) + XCTAssertEqual(copied.toArray(), [1, 2, 3]) + // The 32-bit source keeps its values. + XCTAssertEqual(source.toArray(), [1, 2, 3]) + + let moved = Roaring64Bitmap(moving: source) + XCTAssertEqual(moved.toArray(), [1, 2, 3]) + XCTAssertTrue(source.isEmpty) + } + + func testHashableAndDescription() { + let a: Roaring64Bitmap = [1, 2, 3] + let b: Roaring64Bitmap = [1, 2, 3] + XCTAssertEqual(a.hashValue, b.hashValue) + XCTAssertEqual(Set([a, b]).count, 1) + XCTAssertEqual(a.description, "Roaring64Bitmap(1, 2, 3)") + + let big = Roaring64Bitmap(range: 0..<200, step: 1) + XCTAssertTrue(big.description.hasSuffix(", ...)")) + } +} diff --git a/Tests/swiftRoaringTests/swiftRoaringTests.swift b/Tests/swiftRoaringTests/swiftRoaringTests.swift index 3d75f9e..b38cb14 100644 --- a/Tests/swiftRoaringTests/swiftRoaringTests.swift +++ b/Tests/swiftRoaringTests/swiftRoaringTests.swift @@ -160,7 +160,8 @@ class swiftRoaringTests: XCTestCase { func testSelect() { rbm.addRangeClosed(min: 0, max: 500) - XCTAssertTrue(rbm.select(rank: 5, value: 800)) + XCTAssertEqual(rbm.select(rank: 5), 5) + XCTAssertNil(rbm.select(rank: 501)) XCTAssertEqual(rbm.rank(value: 800), 501) }