Skip to content

Commit e4e9941

Browse files
authored
fix: Fix benchmark results display for paswordless AWS OpenSearch benchmarks (#721)
Fixes #716 Description: When running a benchmark against AWS OpenSearch, username and password are optional and can we left empty so benchmark can use AWS authentiction tokens in environment variables. This works perfectly, and benchamrk runs and does the expected inserts and queries on the AWS OS cluster. But when trying to look at the benchmark results, http://localhost:8502/results displays this error: ``` pydantic.error_wrappers.ValidationError: 2 validation errors for AWSOpenSearchConfig user Empty string! (type=value_error) password Empty string! (type=value_error) File "/Users/jvegas/.cache/uv/archive-v0/DcT73OzatkvN3UXwJBtH_/lib/python3.12/site-packages/vectordb_bench/frontend/pages/results.py", line 62, in <module> main() File "/Users/jvegas/.cache/uv/archive-v0/DcT73OzatkvN3UXwJBtH_/lib/python3.12/site-packages/vectordb_bench/frontend/pages/results.py", line 29, in main allResults = benchmark_runner.get_results() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/jvegas/.cache/uv/archive-v0/DcT73OzatkvN3UXwJBtH_/lib/python3.12/site-packages/vectordb_bench/interface.py", line 106, in get_results return ResultCollector.collect(target_dir) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/jvegas/.cache/uv/archive-v0/DcT73OzatkvN3UXwJBtH_/lib/python3.12/site-packages/vectordb_bench/backend/result_collector.py", line 18, in collect file_result = TestResult.read_file(json_file, trans_unit=True) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/jvegas/.cache/uv/archive-v0/DcT73OzatkvN3UXwJBtH_/lib/python3.12/site-packages/vectordb_bench/models.py", line 331, in read_file task_config["db_config"] = db.config_cls(**task_config["db_config"]) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "pydantic/main.py", line 364, in pydantic.main.BaseModel.__init__ ``` Looks like during results loading the system is applying a more strict validation that does not allow empty username or password, which was allowed at test creation. This PR fixes validation to allow for empty username or password in both cases, resulting in viewables results for passwordless AWS OS benchmarks
1 parent e89c445 commit e4e9941

3 files changed

Lines changed: 87 additions & 7 deletions

File tree

tests/test_aws_opensearch_cli.py

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import pytest
2+
from pydantic import SecretStr
3+
4+
from vectordb_bench.backend.clients.aws_opensearch.cli import optional_secret_str
5+
6+
7+
class TestOptionalSecretStr:
8+
"""Test cases for the optional_secret_str helper function."""
9+
10+
def test_none_input_returns_none(self):
11+
"""Test that None input returns None."""
12+
result = optional_secret_str(None)
13+
assert result is None
14+
15+
def test_string_input_returns_secret_str(self):
16+
"""Test that string input returns SecretStr."""
17+
test_password = "my_secret_password"
18+
result = optional_secret_str(test_password)
19+
20+
assert isinstance(result, SecretStr)
21+
assert result.get_secret_value() == test_password
22+
23+
def test_empty_string_returns_secret_str(self):
24+
"""Test that empty string returns SecretStr with empty value."""
25+
result = optional_secret_str("")
26+
27+
assert isinstance(result, SecretStr)
28+
assert result.get_secret_value() == ""
29+
30+
@pytest.mark.parametrize("test_input,expected_value", [
31+
("password123", "password123"),
32+
("", ""),
33+
("special!@#$%^&*()chars", "special!@#$%^&*()chars"),
34+
(" spaces ", " spaces "),
35+
("unicode_ñáéíóú", "unicode_ñáéíóú"),
36+
])
37+
def test_various_string_inputs(self, test_input, expected_value):
38+
"""Test various string inputs return SecretStr with correct values."""
39+
result = optional_secret_str(test_input)
40+
41+
assert isinstance(result, SecretStr)
42+
assert result.get_secret_value() == expected_value
43+
44+
def test_none_vs_empty_string_difference(self):
45+
"""Test that None and empty string are handled differently."""
46+
none_result = optional_secret_str(None)
47+
empty_result = optional_secret_str("")
48+
49+
assert none_result is None
50+
assert isinstance(empty_result, SecretStr)
51+
assert empty_result.get_secret_value() == ""
52+
53+
def test_return_type_annotations(self):
54+
"""Test that return types match the function signature."""
55+
# Test None case
56+
none_result = optional_secret_str(None)
57+
assert none_result is None
58+
59+
# Test string case
60+
string_result = optional_secret_str("test")
61+
assert isinstance(string_result, SecretStr)

vectordb_bench/backend/clients/aws_opensearch/cli.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,16 @@
1717
log = logging.getLogger(__name__)
1818

1919

20+
def optional_secret_str(value: str | None) -> SecretStr | None:
21+
"""Convert string to SecretStr, handling None gracefully."""
22+
return None if value is None else SecretStr(value)
23+
24+
2025
class AWSOpenSearchTypedDict(TypedDict):
2126
host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)]
2227
port: Annotated[int, click.option("--port", type=int, default=80, help="Db Port")]
23-
user: Annotated[str, click.option("--user", type=str, help="Db User")]
24-
password: Annotated[str, click.option("--password", type=str, help="Db password")]
28+
user: Annotated[str | None, click.option("--user", type=str, help="Db User")]
29+
password: Annotated[str | None, click.option("--password", type=str, help="Db password")]
2530
number_of_shards: Annotated[
2631
int,
2732
click.option("--number-of-shards", type=int, help="Number of primary shards for the index", default=1),
@@ -188,7 +193,7 @@ def AWSOpenSearch(**parameters: Unpack[AWSOpenSearchHNSWTypedDict]):
188193
host=parameters["host"],
189194
port=parameters["port"],
190195
user=parameters["user"],
191-
password=SecretStr(parameters["password"]),
196+
password=optional_secret_str(parameters["password"]),
192197
),
193198
db_case_config=AWSOpenSearchIndexConfig(
194199
number_of_shards=parameters["number_of_shards"],

vectordb_bench/backend/clients/aws_opensearch/config.py

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import logging
22
from enum import Enum
33

4-
from pydantic import BaseModel, SecretStr
4+
from pydantic import BaseModel, SecretStr, validator
55

66
from ..api import DBCaseConfig, DBConfig, MetricType
77

@@ -11,13 +11,15 @@
1111
class AWSOpenSearchConfig(DBConfig, BaseModel):
1212
host: str = ""
1313
port: int = 80
14-
user: str = ""
15-
password: SecretStr = ""
14+
user: str | None = None
15+
password: SecretStr | None = None
1616

1717
def to_dict(self) -> dict:
1818
use_ssl = self.port == 443
1919
http_auth = (
20-
(self.user, self.password.get_secret_value()) if len(self.user) != 0 and len(self.password) != 0 else ()
20+
(self.user, self.password.get_secret_value())
21+
if self.user is not None and self.password is not None and len(self.user) != 0 and len(self.password) != 0
22+
else ()
2123
)
2224
return {
2325
"hosts": [{"host": self.host, "port": self.port}],
@@ -30,6 +32,18 @@ def to_dict(self) -> dict:
3032
"timeout": 600,
3133
}
3234

35+
@validator("*")
36+
def not_empty_field(cls, v: any, field: any):
37+
if (
38+
field.name in cls.common_short_configs()
39+
or field.name in cls.common_long_configs()
40+
or field.name in ["user", "password", "host"]
41+
):
42+
return v
43+
if isinstance(v, str | SecretStr) and len(v) == 0:
44+
raise ValueError("Empty string!")
45+
return v
46+
3347

3448
class AWSOS_Engine(Enum):
3549
faiss = "faiss"

0 commit comments

Comments
 (0)