From 5c6d73ffe51a50646e411507d0044e1b38a3f753 Mon Sep 17 00:00:00 2001 From: Harsh Thakare Date: Fri, 7 Aug 2026 14:37:29 +0530 Subject: [PATCH 1/2] Use native factorization for object string categories --- python/xy/channels.py | 37 ++++++++++++++++---- tests/test_custom_ramps_and_palette.py | 47 ++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/python/xy/channels.py b/python/xy/channels.py index 323ed5e0..b152d1e1 100644 --- a/python/xy/channels.py +++ b/python/xy/channels.py @@ -371,6 +371,25 @@ def _use_native_fixed_factorizer(arr: np.ndarray) -> bool: return distinct < near_unique * len(probe) +def _normalize_object_strings(arr: np.ndarray) -> Optional[np.ndarray]: + """Return object-backed strings as fixed-width Unicode when safe. + + Object arrays can contain values with display-label semantics that cannot + be represented by a simple Unicode cast. Restrict this path to exact + Python strings so mixed objects, missing values, and custom ``__str__`` + implementations retain the canonical fallback behavior. + """ + if arr.dtype.kind != "O" or arr.size == 0: + return None + values = arr.reshape(-1) + width = 0 + for value in values: + if type(value) is not str: + return None + width = max(width, len(value)) + return np.asarray(arr, dtype=f" tuple[ @@ -388,15 +407,19 @@ def _factorize_categories( strings/bytes/bools can identify equal records in Rust without creating N Python objects; only their compact unique set crosses the label-policy path. """ - if arr.dtype.kind in ("U", "S", "b") and _use_native_fixed_factorizer(arr): + normalized = _normalize_object_strings(arr) + factorizer_arr = arr if normalized is None else normalized + if factorizer_arr.dtype.kind in ("U", "S", "b") and _use_native_fixed_factorizer( + factorizer_arr + ): compact = ( - kernels.factorize_unicode1_u8_counts(arr, MAX_CATEGORIES) - if arr.dtype.kind == "U" and arr.dtype.itemsize == 4 - else kernels.factorize_fixed_u8_counts(arr, MAX_CATEGORIES) + kernels.factorize_unicode1_u8_counts(factorizer_arr, MAX_CATEGORIES) + if factorizer_arr.dtype.kind == "U" and factorizer_arr.dtype.itemsize == 4 + else kernels.factorize_fixed_u8_counts(factorizer_arr, MAX_CATEGORIES) ) if compact is not None: raw_codes, unique_indices, raw_counts = compact - unique_labels = [category_label(value) for value in arr[unique_indices]] + unique_labels = [category_label(value) for value in factorizer_arr[unique_indices]] categories = sorted(set(unique_labels)) index = {label: i for i, label in enumerate(categories)} remap = np.fromiter( @@ -412,8 +435,8 @@ def _factorize_categories( counts[index[label]] += count return categories, raw_codes, counts - raw_codes, unique_indices = kernels.factorize_fixed(arr) - unique_labels = [category_label(value) for value in arr[unique_indices]] + raw_codes, unique_indices = kernels.factorize_fixed(factorizer_arr) + unique_labels = [category_label(value) for value in factorizer_arr[unique_indices]] categories = sorted(set(unique_labels)) index = {label: i for i, label in enumerate(categories)} dtype = _category_code_dtype(len(categories)) diff --git a/tests/test_custom_ramps_and_palette.py b/tests/test_custom_ramps_and_palette.py index 31764736..d6ee7c67 100644 --- a/tests/test_custom_ramps_and_palette.py +++ b/tests/test_custom_ramps_and_palette.py @@ -375,6 +375,53 @@ def test_the_literal_color_probe_does_not_materialize_category_columns(): assert channels._literal_color_rgba(column) is None +def test_homogeneous_object_categories_use_native_factorization(monkeypatch): + """Object strings use the native path without an N-entry label list.""" + values = np.array(["group-a", "group-b", "group-a", "group-c"] * 250, dtype=object) + seen: list[object] = [] + original = channels.category_label + + def record(value: object) -> str: + seen.append(value) + return original(value) + + monkeypatch.setattr(channels, "category_label", record) + categories, codes, counts = channels._factorize_categories(values) + + assert categories == ["group-a", "group-b", "group-c"] + np.testing.assert_array_equal(codes[:4], [0, 1, 0, 2]) + np.testing.assert_array_equal(counts, [500, 250, 250]) + assert len(seen) == len(categories) + + +def test_object_factorization_matches_fixed_width_string_semantics(): + """The object fast path preserves category order, codes, and counts.""" + values = np.array(["beta", "alpha", "beta", "gamma", "alpha"], dtype=object) + object_result = channels._factorize_categories(values) + unicode_result = channels._factorize_categories(values.astype("U5")) + + assert object_result[0] == unicode_result[0] + np.testing.assert_array_equal(object_result[1], unicode_result[1]) + np.testing.assert_array_equal(object_result[2], unicode_result[2]) + + +@pytest.mark.parametrize( + "values", + [ + np.array(["a", None, "a"], dtype=object), + np.array(["a", 1, "a"], dtype=object), + np.array([b"a", b"b", b"a"], dtype=object), + ], +) +def test_mixed_object_categories_keep_the_fallback(values): + """Values without exact string semantics remain on the safe fallback.""" + categories, codes, counts = channels._factorize_categories(values) + + assert len(categories) == len(set(categories)) + assert len(codes) == len(values) + assert counts is None + + def test_the_probe_still_reads_a_column_that_actually_looks_like_paint(): column = np.array(["#ff0000", "#00ff00", "#0000ff"]) rgba = channels._literal_color_rgba(column) From f1c9c6ae94d2bfc075493dda3b5c25f94fa9b841 Mon Sep 17 00:00:00 2001 From: Harsh Thakare Date: Fri, 7 Aug 2026 15:28:08 +0530 Subject: [PATCH 2/2] Address object factorizer review feedback --- python/xy/channels.py | 62 +++++++++++++++++++------- tests/test_custom_ramps_and_palette.py | 27 +++++++++++ 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/python/xy/channels.py b/python/xy/channels.py index b152d1e1..194bdc16 100644 --- a/python/xy/channels.py +++ b/python/xy/channels.py @@ -342,7 +342,18 @@ def _category_code_dtype(category_count: int) -> type[np.uint8] | type[np.uint32 return np.uint8 if category_count <= MAX_CATEGORIES else np.uint32 -def _use_native_fixed_factorizer(arr: np.ndarray) -> bool: +def _factorize_probe(arr: np.ndarray) -> np.ndarray: + """Return the bounded sample used to select the native factorizer.""" + n = len(arr) + if n <= _FACTORIZE_PROBE_ROWS: + return arr + rows = np.linspace(0, n - 1, _FACTORIZE_PROBE_ROWS, dtype=np.intp) + return arr[rows] + + +def _use_native_fixed_factorizer( + arr: np.ndarray, *, normalized_itemsize: Optional[int] = None +) -> bool: """Choose the O(N) hash path unless a bounded global probe says it cannot pay. The native pass earns its keep by keeping N records out of Python: only the @@ -356,37 +367,40 @@ def _use_native_fixed_factorizer(arr: np.ndarray) -> bool: repeats get scarce while narrow ones hold it until the probe is entirely distinct. Sampling across the full array keeps the decision independent of N. """ - n = len(arr) - if n <= _FACTORIZE_PROBE_ROWS: - probe = arr - else: - rows = np.linspace(0, n - 1, _FACTORIZE_PROBE_ROWS, dtype=np.intp) - probe = arr[rows] + probe = _factorize_probe(arr) distinct = len(np.unique(probe)) if distinct <= _FACTORIZE_NATIVE_MAX_PROBE_CATEGORIES: return True near_unique = ( - 1.0 if arr.dtype.itemsize <= _FACTORIZE_NARROW_ITEMSIZE else _FACTORIZE_NEAR_UNIQUE_RATIO + 1.0 + if (normalized_itemsize or arr.dtype.itemsize) <= _FACTORIZE_NARROW_ITEMSIZE + else _FACTORIZE_NEAR_UNIQUE_RATIO ) return distinct < near_unique * len(probe) -def _normalize_object_strings(arr: np.ndarray) -> Optional[np.ndarray]: - """Return object-backed strings as fixed-width Unicode when safe. +def _object_string_width(arr: np.ndarray, *, sample: bool = False) -> Optional[int]: + """Return the width of safe Python-string values, or ``None``. Object arrays can contain values with display-label semantics that cannot be represented by a simple Unicode cast. Restrict this path to exact Python strings so mixed objects, missing values, and custom ``__str__`` - implementations retain the canonical fallback behavior. + implementations retain the canonical fallback behavior. NUL codepoints + are excluded because NumPy treats them as fixed-width string padding. """ if arr.dtype.kind != "O" or arr.size == 0: return None - values = arr.reshape(-1) + values = _factorize_probe(arr) if sample else arr.reshape(-1) width = 0 for value in values: - if type(value) is not str: + if type(value) is not str or "\x00" in value: return None width = max(width, len(value)) + return width + + +def _normalize_object_strings(arr: np.ndarray, width: int) -> np.ndarray: + """Normalize validated object-backed strings to fixed-width Unicode.""" return np.asarray(arr, dtype=f"