Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions redisvl/redis/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -405,6 +405,7 @@ def parse_vector_attrs(attrs):
# params in "flags" as key/value pairs, possibly followed by
# valueless modifiers: ["algorithm", "FLAT", "dim", 4, "INDEXMISSING"]
flags = attrs.get("flags") or []
index_missing = "INDEXMISSING" in flags
i = 0
while i < len(flags):
key = str(flags[i]).lower()
Expand All @@ -414,6 +415,10 @@ def parse_vector_attrs(attrs):
vector_attrs.setdefault(key, flags[i + 1])
i += 2
else:
# INDEXMISSING is a valueless modifier after the key/value pairs
index_missing = "INDEXMISSING" in attrs[6:]
attrs = attrs[:6] + [a for a in attrs[6:] if a != "INDEXMISSING"]

# Check if we have any attributes beyond the type declaration
if len(attrs) <= 6:
# Redis 6.2.6-v9 or similar: no vector params in FT.INFO
Expand Down Expand Up @@ -561,6 +566,9 @@ def parse_vector_attrs(attrs):
except (ValueError, TypeError):
pass

if index_missing:
normalized["index_missing"] = True

# Validate that we have required dims
if "dims" not in normalized:
# Could not parse dims - this field is not properly supported
Expand Down
26 changes: 21 additions & 5 deletions redisvl/schema/fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,8 +229,6 @@ def field_data(self) -> dict[str, Any]:
}
if self.initial_cap is not None: # Only include it if it's set
field_data["INITIAL_CAP"] = self.initial_cap
if self.index_missing: # Only include it if it's set
field_data["INDEXMISSING"] = True
return field_data


Expand Down Expand Up @@ -557,7 +555,13 @@ def as_redis_field(self) -> RedisField:
field_data = self.attrs.field_data
if self.attrs.block_size is not None:
field_data["BLOCK_SIZE"] = self.attrs.block_size
return RedisVectorField(name, self.attrs.algorithm, field_data, as_name=as_name)
return RedisVectorField(
name,
self.attrs.algorithm,
field_data,
as_name=as_name,
index_missing=self.attrs.index_missing,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now that a vector field can be created with index_missing, from_existing should recover it too. Otherwise a from_existing → create(overwrite=True) round trip still quietly drops the modifier. The parser skips the trailing INDEXMISSING in the vector branch of convert_index_info_to_schema (redisvl/redis/connection.py, the flags loop around line 408), and tests/unit/test_convert_index_info.py:277-279 pins that, with a comment that's now out of date ("can't be recreated with it yet"). Could you pick it up there, flip that assertion to expect index_missing: True, and check it against the four client configs in test_client_from_existing_and_batch_search?

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in 064fd78. parse_vector_attrs now sets index_missing: True when INDEXMISSING follows the vector params. That covers the trailing token in the list response and flags in the dict responses, including the packed RESP2 form. The assertion in test_convert_index_info.py now expects index_missing: True, and the outdated comment is gone.

The vector field in test_client_from_existing_and_batch_search (sync and async) now sets index_missing: True, so its reopened.schema == index.schema check covers the round trip. All four client configs pass against redis:8.4 with redis-py 8.1. Without the parser change, all four fail.

)


class HNSWVectorField(BaseField):
Expand All @@ -578,7 +582,13 @@ def as_redis_field(self) -> RedisField:
"EPSILON": self.attrs.epsilon,
}
)
return RedisVectorField(name, self.attrs.algorithm, field_data, as_name=as_name)
return RedisVectorField(
name,
self.attrs.algorithm,
field_data,
as_name=as_name,
index_missing=self.attrs.index_missing,
)


class SVSVectorField(BaseField):
Expand All @@ -605,7 +615,13 @@ def as_redis_field(self) -> RedisField:
field_data["REDUCE"] = self.attrs.reduce
if self.attrs.training_threshold is not None:
field_data["TRAINING_THRESHOLD"] = self.attrs.training_threshold
return RedisVectorField(name, self.attrs.algorithm, field_data, as_name=as_name)
return RedisVectorField(
name,
self.attrs.algorithm,
field_data,
as_name=as_name,
index_missing=self.attrs.index_missing,
)


FIELD_TYPE_MAP = {
Expand Down
1 change: 1 addition & 0 deletions tests/integration/test_async_search_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -582,6 +582,7 @@ async def test_client_from_existing_and_batch_search(
"distance_metric": "cosine",
"algorithm": "flat",
"datatype": "float32",
"index_missing": True,
},
},
],
Expand Down
24 changes: 19 additions & 5 deletions tests/integration/test_field_modifier_ordering_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,21 @@ def test_index_creation_with_multiple_modifiers(self, client, redis_url, worker_
# Cleanup
index.delete(drop=True)

def test_indexmissing_enables_ismissing_query(self, client, redis_url, worker_id):
@pytest.mark.parametrize(
"field_type,attrs,value",
[
pytest.param("text", {}, "has value", id="text"),
pytest.param(
"vector",
{"algorithm": "hnsw", "dims": 3},
b"\x00" * 12,
id="vector-hnsw",
),
],
)
def test_indexmissing_enables_ismissing_query(
self, client, redis_url, worker_id, field_type, attrs, value
):
"""Test that INDEXMISSING enables ismissing() query function."""
skip_if_search_version_below_for_indexmissing(client)
schema_dict = {
Expand All @@ -219,8 +233,8 @@ def test_indexmissing_enables_ismissing_query(self, client, redis_url, worker_id
"fields": [
{
"name": "optional_field",
"type": "text",
"attrs": {"index_missing": True},
"type": field_type,
"attrs": {**attrs, "index_missing": True},
}
],
}
Expand All @@ -231,9 +245,9 @@ def test_indexmissing_enables_ismissing_query(self, client, redis_url, worker_id
index.create(overwrite=True)

# Create documents with and without the field
client.hset(f"ismiss_{worker_id}:1", "optional_field", "has value")
client.hset(f"ismiss_{worker_id}:1", "optional_field", value)
client.hset(f"ismiss_{worker_id}:2", "other_field", "no optional_field")
client.hset(f"ismiss_{worker_id}:3", "optional_field", "also has value")
client.hset(f"ismiss_{worker_id}:3", "optional_field", value)

# Query for missing fields
result = client.execute_command(
Expand Down
1 change: 1 addition & 0 deletions tests/integration/test_search_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -645,6 +645,7 @@ def test_client_from_existing_and_batch_search(
"distance_metric": "cosine",
"algorithm": "flat",
"datatype": "float32",
"index_missing": True,
},
},
],
Expand Down
4 changes: 1 addition & 3 deletions tests/unit/test_convert_index_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,9 +274,6 @@ def test_convert_index_info_resp2_unified_vector_flags():

result = convert_index_info_to_schema(index_info)

# INDEXMISSING is deliberately not recovered as ``index_missing``: a vector
# field can't be recreated with it yet, so restoring it here would turn a
# quiet loss into a failure on ``create()``.
assert result["fields"] == [
{
"name": "emb",
Expand All @@ -288,6 +285,7 @@ def test_convert_index_info_resp2_unified_vector_flags():
"datatype": "float32",
"m": 24,
"ef_construction": 300,
"index_missing": True,
},
}
]
Expand Down
68 changes: 17 additions & 51 deletions tests/unit/test_fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -356,60 +356,26 @@ def test_redis_field_creation_with_index_attributes(
# We can't directly inspect them, but we can verify the field creation doesn't fail


def test_vector_fields_redis_creation_with_index_missing():
"""Test that vector fields properly handle index_missing in Redis field creation."""

# Test FlatVectorField with index_missing
flat_field = FlatVectorField(
name="flat_embedding",
attrs={
"algorithm": "flat",
"dims": 128,
"index_missing": True,
"block_size": 100,
},
)
redis_field = flat_field.as_redis_field()
assert isinstance(redis_field, RedisVectorField)
assert redis_field.name == "flat_embedding"

# Test HNSWVectorField with index_missing
hnsw_field = HNSWVectorField(
name="hnsw_embedding",
attrs={"algorithm": "hnsw", "dims": 256, "index_missing": True, "m": 24},
)
redis_field = hnsw_field.as_redis_field()
assert isinstance(redis_field, RedisVectorField)
assert redis_field.name == "hnsw_embedding"


def test_vector_field_data_includes_index_missing():
"""Test that vector field field_data includes INDEXMISSING when enabled."""

# Test with index_missing=True
flat_field_with_missing = FlatVectorField(
@pytest.mark.parametrize(
"field_class,algorithm",
[
(FlatVectorField, "flat"),
(HNSWVectorField, "hnsw"),
(SVSVectorField, "svs-vamana"),
],
)
def test_vector_field_index_missing_follows_attribute_block(field_class, algorithm):
"""INDEXMISSING is a field modifier, so it must follow the vector attribute
block rather than be counted and sent inside it."""
field = field_class(
name="embedding",
attrs={"algorithm": "flat", "dims": 128, "index_missing": True},
attrs={"algorithm": algorithm, "dims": 128, "index_missing": True},
)
field_data = flat_field_with_missing.attrs.field_data
assert "INDEXMISSING" in field_data
assert field_data["INDEXMISSING"] == True

# Test with index_missing=False (default)
flat_field_without_missing = FlatVectorField(
name="embedding", attrs={"algorithm": "flat", "dims": 128}
)
field_data = flat_field_without_missing.attrs.field_data
assert "INDEXMISSING" not in field_data

# Test HNSW field with index_missing=True
hnsw_field_with_missing = HNSWVectorField(
name="embedding",
attrs={"algorithm": "hnsw", "dims": 256, "index_missing": True},
)
field_data = hnsw_field_with_missing.attrs.field_data
assert "INDEXMISSING" in field_data
assert field_data["INDEXMISSING"] == True
args = field.as_redis_field().redis_args()
attr_count = args[3]
assert "INDEXMISSING" not in args[: 4 + attr_count]
assert args[4 + attr_count :] == ["INDEXMISSING"]


def test_field_factory_with_new_attributes():
Expand Down