From 1ffd9c26d06455f845f678b8d39b311c43e9710a Mon Sep 17 00:00:00 2001 From: Anubhav Dhawan Date: Thu, 20 Aug 2026 12:08:44 +0530 Subject: [PATCH] test(core): add end-to-end integration tests for secure parameters --- .../toolbox-adk/integration.cloudbuild.yaml | 2 +- .../tests/integration/test_integration.py | 104 +++++++++++- .../toolbox-core/integration.cloudbuild.yaml | 2 +- packages/toolbox-core/tests/test_e2e.py | 9 +- packages/toolbox-core/tests/test_e2e_mcp.py | 160 +++++++++++++++++- packages/toolbox-core/tests/test_sync_e2e.py | 46 +++++ .../integration.cloudbuild.yaml | 2 +- packages/toolbox-langchain/tests/test_e2e.py | 52 ++++-- .../integration.cloudbuild.yaml | 2 +- packages/toolbox-llamaindex/tests/test_e2e.py | 52 ++++-- 10 files changed, 402 insertions(+), 29 deletions(-) diff --git a/packages/toolbox-adk/integration.cloudbuild.yaml b/packages/toolbox-adk/integration.cloudbuild.yaml index 6c1bfc2b1..801cf0333 100644 --- a/packages/toolbox-adk/integration.cloudbuild.yaml +++ b/packages/toolbox-adk/integration.cloudbuild.yaml @@ -51,4 +51,4 @@ substitutions: _VERSION: '3.13' # Default values (can be overridden by triggers) _TOOLBOX_VERSION: '1.9.0' - _TOOLBOX_MANIFEST_VERSION: '34' + _TOOLBOX_MANIFEST_VERSION: '38' diff --git a/packages/toolbox-adk/tests/integration/test_integration.py b/packages/toolbox-adk/tests/integration/test_integration.py index ebc0e4bd7..1ae91baac 100644 --- a/packages/toolbox-adk/tests/integration/test_integration.py +++ b/packages/toolbox-adk/tests/integration/test_integration.py @@ -417,7 +417,6 @@ async def test_load_toolset_default(self): ) try: tools = await toolset.get_tools() - assert len(tools) == 7 tool_names = {tool.name for tool in tools} expected_tools = [ "get-row-by-content-auth", @@ -427,7 +426,9 @@ async def test_load_toolset_default(self): "get-n-rows", "search-rows", "process-data", + "my-secure-tool", ] + assert len(tools) == len(expected_tools) assert tool_names == set(expected_tools) finally: await toolset.close() @@ -920,3 +921,104 @@ async def __call__(self, my_array=None, my_object=None, **kwargs): assert event_count > 0 assert success, "Agent failed to use the tool successfully" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("toolbox_server") +class TestSecureParamsE2E: + """End-to-end integration tests for ADK with secure parameters.""" + + async def test_adk_toolset_with_secure_params(self): + """Tests ToolboxToolset loading by toolset_name and running tools with secure parameters.""" + toolset = ToolboxToolset( + server_url=TOOLBOX_SERVER_URL_STABLE, + toolset_name="my-secure-toolset", + credentials=CredentialStrategy.toolbox_identity(), + secure_params={"name": "Alice"}, + ) + try: + tools = await toolset.get_tools() + by_name = {t.name: t for t in tools} + assert "my-secure-tool" in by_name + tool = by_name["my-secure-tool"] + ctx = MagicMock() + result = await tool.run_async({"id": 1}, ctx) + assert isinstance(result, str) + assert "Alice" in result + finally: + await toolset.close() + + async def test_adk_tool_bind_secure_param(self): + """Tests binding a secure parameter on an individual ADK ToolboxTool.""" + toolset = ToolboxToolset( + server_url=TOOLBOX_SERVER_URL_STABLE, + toolset_name="my-secure-toolset", + credentials=CredentialStrategy.toolbox_identity(), + ) + try: + tools = await toolset.get_tools() + by_name = {t.name: t for t in tools} + assert "my-secure-tool" in by_name + tool = by_name["my-secure-tool"] + bound_tool = tool.bind_secure_param("name", "Alice") + ctx = MagicMock() + result = await bound_tool.run_async({"id": 1}, ctx) + assert isinstance(result, str) + assert "Alice" in result + finally: + await toolset.close() + + async def test_adk_dynamic_callable_re_evaluation_per_invocation(self): + """Tests that dynamic callables are re-evaluated per invocation in ADK.""" + counter = 0 + + def dynamic_name(): + nonlocal counter + counter += 1 + return f"User{counter}" + + toolset = ToolboxToolset( + server_url=TOOLBOX_SERVER_URL_STABLE, + toolset_name="my-secure-toolset", + credentials=CredentialStrategy.toolbox_identity(), + secure_params={"name": dynamic_name}, + ) + try: + tools = await toolset.get_tools() + by_name = {t.name: t for t in tools} + assert "my-secure-tool" in by_name + tool = by_name["my-secure-tool"] + ctx = MagicMock() + + # First invocation -> counter = 1 -> "User1" + res1 = await tool.run_async({"id": 1}, ctx) + assert isinstance(res1, str) + assert "User1" in res1 + + # Second invocation -> counter = 2 -> "User2" + res2 = await tool.run_async({"id": 1}, ctx) + assert isinstance(res2, str) + assert "User2" in res2 + finally: + await toolset.close() + + async def test_adk_secure_param_declaration_isolation(self): + """Tests that secure parameters are excluded from ADK Gemini function declaration.""" + toolset = ToolboxToolset( + server_url=TOOLBOX_SERVER_URL_STABLE, + toolset_name="my-secure-toolset", + credentials=CredentialStrategy.toolbox_identity(), + ) + try: + tools = await toolset.get_tools() + by_name = {t.name: t for t in tools} + assert "my-secure-tool" in by_name + tool = by_name["my-secure-tool"] + declaration = tool._get_declaration() + assert declaration is not None + assert declaration.parameters is not None + assert hasattr(declaration.parameters, "properties") + assert "id" in declaration.parameters.properties + assert "name" not in declaration.parameters.properties + finally: + await toolset.close() diff --git a/packages/toolbox-core/integration.cloudbuild.yaml b/packages/toolbox-core/integration.cloudbuild.yaml index 0c18c43a3..83fab86f6 100644 --- a/packages/toolbox-core/integration.cloudbuild.yaml +++ b/packages/toolbox-core/integration.cloudbuild.yaml @@ -46,4 +46,4 @@ options: substitutions: _VERSION: '3.13' _TOOLBOX_VERSION: '1.9.0' - _TOOLBOX_MANIFEST_VERSION: '34' + _TOOLBOX_MANIFEST_VERSION: '38' diff --git a/packages/toolbox-core/tests/test_e2e.py b/packages/toolbox-core/tests/test_e2e.py index f5a0ebc5c..afaa51499 100644 --- a/packages/toolbox-core/tests/test_e2e.py +++ b/packages/toolbox-core/tests/test_e2e.py @@ -75,7 +75,6 @@ async def test_load_toolset_specific( async def test_load_toolset_default(self, toolbox: ToolboxClient): """Load the default toolset, i.e. all tools.""" toolset = await toolbox.load_toolset() - assert len(toolset) == 7 tool_names = {tool.__name__ for tool in toolset} expected_tools = [ "get-row-by-content-auth", @@ -86,6 +85,14 @@ async def test_load_toolset_default(self, toolbox: ToolboxClient): "search-rows", "process-data", ] + + protocol_version = toolbox._ToolboxClient__transport._protocol_version + if Protocol._is_version_at_least( + protocol_version, Protocol.MCP_v20260728.value + ): + expected_tools.append("my-secure-tool") + + assert len(toolset) == len(expected_tools) assert tool_names == set(expected_tools) async def test_run_tool(self, get_n_rows_tool: ToolboxTool): diff --git a/packages/toolbox-core/tests/test_e2e_mcp.py b/packages/toolbox-core/tests/test_e2e_mcp.py index 1924498b4..811d72962 100644 --- a/packages/toolbox-core/tests/test_e2e_mcp.py +++ b/packages/toolbox-core/tests/test_e2e_mcp.py @@ -77,7 +77,6 @@ async def test_load_toolset_specific( async def test_load_toolset_default(self, toolbox: ToolboxClient): """Load the default toolset, i.e. all tools.""" toolset = await toolbox.load_toolset() - assert len(toolset) == 7 tool_names = {tool.__name__ for tool in toolset} expected_tools = [ "get-row-by-content-auth", @@ -88,6 +87,14 @@ async def test_load_toolset_default(self, toolbox: ToolboxClient): "search-rows", "process-data", ] + + protocol_version = toolbox._ToolboxClient__transport._protocol_version + if Protocol._is_version_at_least( + protocol_version, Protocol.MCP_v20260728.value + ): + expected_tools.append("my-secure-tool") + + assert len(toolset) == len(expected_tools) assert tool_names == set(expected_tools) async def test_run_tool(self, get_n_rows_tool: ToolboxTool): @@ -590,3 +597,154 @@ async def test_mcp_custom_protocols_list(toolbox_server_url: str): client._ToolboxClient__transport._protocol_version == Protocol.MCP_DRAFT.value ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("toolbox_server") +class TestSecureParamsE2E: + async def test_run_tool_with_secure_param(self, toolbox: ToolboxClient): + """Tests loading and invoking a tool with a secure parameter.""" + protocol_version = toolbox._ToolboxClient__transport._protocol_version + if not Protocol._is_version_at_least( + protocol_version, Protocol.MCP_v20260728.value + ): + with pytest.raises(ValueError, match="Tool my-secure-tool not found"): + await toolbox.load_tool("my-secure-tool") + return + + tool = await toolbox.load_tool("my-secure-tool") + bound_tool = tool.bind_secure_param("name", "Alice") + response = await bound_tool(id=1) + assert isinstance(response, str) + assert "Alice" in response + + async def test_run_tool_with_secure_params_plural(self, toolbox: ToolboxClient): + """Tests batch binding with bind_secure_params.""" + protocol_version = toolbox._ToolboxClient__transport._protocol_version + if not Protocol._is_version_at_least( + protocol_version, Protocol.MCP_v20260728.value + ): + with pytest.raises(ValueError, match="Tool my-secure-tool not found"): + await toolbox.load_tool("my-secure-tool") + return + + tool = await toolbox.load_tool("my-secure-tool") + bound_tool = tool.bind_secure_params({"name": "Alice"}) + response = await bound_tool(id=1) + assert isinstance(response, str) + assert "Alice" in response + + async def test_run_tool_with_secure_param_callable_sync( + self, toolbox: ToolboxClient + ): + """Tests dynamic sync callable resolution during live tool execution.""" + protocol_version = toolbox._ToolboxClient__transport._protocol_version + if not Protocol._is_version_at_least( + protocol_version, Protocol.MCP_v20260728.value + ): + with pytest.raises(ValueError, match="Tool my-secure-tool not found"): + await toolbox.load_tool("my-secure-tool") + return + + tool = await toolbox.load_tool("my-secure-tool") + bound_tool = tool.bind_secure_param("name", lambda: "Alice") + response = await bound_tool(id=1) + assert isinstance(response, str) + assert "Alice" in response + + async def test_run_tool_with_secure_param_callable_async( + self, toolbox: ToolboxClient + ): + """Tests dynamic async coroutine resolution during live tool execution.""" + protocol_version = toolbox._ToolboxClient__transport._protocol_version + if not Protocol._is_version_at_least( + protocol_version, Protocol.MCP_v20260728.value + ): + with pytest.raises(ValueError, match="Tool my-secure-tool not found"): + await toolbox.load_tool("my-secure-tool") + return + + tool = await toolbox.load_tool("my-secure-tool") + + async def fetch_secret(): + return "Alice" + + bound_tool = tool.bind_secure_param("name", fetch_secret) + response = await bound_tool(id=1) + assert isinstance(response, str) + assert "Alice" in response + + async def test_secure_param_callable_exception_propagates( + self, toolbox: ToolboxClient + ): + """Tests that exceptions in dynamic callables propagate to caller.""" + protocol_version = toolbox._ToolboxClient__transport._protocol_version + if not Protocol._is_version_at_least( + protocol_version, Protocol.MCP_v20260728.value + ): + with pytest.raises(ValueError, match="Tool my-secure-tool not found"): + await toolbox.load_tool("my-secure-tool") + return + + tool = await toolbox.load_tool("my-secure-tool") + + def failing_secret(): + raise PermissionError("token expired") + + bound_tool = tool.bind_secure_param("name", failing_secret) + with pytest.raises(PermissionError, match="token expired"): + await bound_tool(id=1) + + async def test_load_tool_with_secure_params(self, toolbox: ToolboxClient): + """Tests load_tool with secure_params passed during loading.""" + protocol_version = toolbox._ToolboxClient__transport._protocol_version + if not Protocol._is_version_at_least( + protocol_version, Protocol.MCP_v20260728.value + ): + with pytest.raises(ValueError, match="Tool my-secure-tool not found"): + await toolbox.load_tool( + "my-secure-tool", secure_params={"name": "Alice"} + ) + return + + tool = await toolbox.load_tool( + "my-secure-tool", secure_params={"name": "Alice"} + ) + response = await tool(id=1) + assert isinstance(response, str) + assert "Alice" in response + + async def test_load_toolset_with_secure_params(self, toolbox: ToolboxClient): + """Tests load_toolset with secure_params distributed across tools.""" + protocol_version = toolbox._ToolboxClient__transport._protocol_version + toolset = await toolbox.load_toolset( + "my-secure-toolset", secure_params={"name": "Alice"} + ) + if not Protocol._is_version_at_least( + protocol_version, Protocol.MCP_v20260728.value + ): + assert len(toolset) == 0 + return + + by_name = {t.__name__: t for t in toolset} + assert "my-secure-tool" in by_name + tool = by_name["my-secure-tool"] + response = await tool(id=1) + assert isinstance(response, str) + assert "Alice" in response + + async def test_secure_param_schema_isolation_e2e(self, toolbox: ToolboxClient): + """Tests that secure parameters from server are stripped from __signature__ and docstring.""" + protocol_version = toolbox._ToolboxClient__transport._protocol_version + if not Protocol._is_version_at_least( + protocol_version, Protocol.MCP_v20260728.value + ): + with pytest.raises(ValueError, match="Tool my-secure-tool not found"): + await toolbox.load_tool("my-secure-tool") + return + + tool = await toolbox.load_tool("my-secure-tool") + sig = signature(tool) + assert "id" in sig.parameters + assert "name" not in sig.parameters + assert "name" not in (tool.__doc__ or "") diff --git a/packages/toolbox-core/tests/test_sync_e2e.py b/packages/toolbox-core/tests/test_sync_e2e.py index 4d0f43f2f..e1f5cfbad 100644 --- a/packages/toolbox-core/tests/test_sync_e2e.py +++ b/packages/toolbox-core/tests/test_sync_e2e.py @@ -201,3 +201,49 @@ def test_run_tool_param_auth_no_field( ) response = tool() assert "no field named row_data in claims" in response + + +@pytest.mark.usefixtures("toolbox_server") +class TestSyncSecureParamsE2E: + def test_sync_run_tool_with_secure_param(self, toolbox: ToolboxSyncClient): + """Tests synchronous loading and invoking a tool with a secure parameter.""" + tool = toolbox.load_tool("my-secure-tool") + bound_tool = tool.bind_secure_param("name", "Alice") + response = bound_tool(id=1) + assert isinstance(response, str) + assert "Alice" in response + + def test_sync_run_tool_with_secure_params_plural(self, toolbox: ToolboxSyncClient): + """Tests synchronous batch binding with bind_secure_params.""" + tool = toolbox.load_tool("my-secure-tool") + bound_tool = tool.bind_secure_params({"name": "Alice"}) + response = bound_tool(id=1) + assert isinstance(response, str) + assert "Alice" in response + + def test_sync_run_tool_with_secure_param_callable(self, toolbox: ToolboxSyncClient): + """Tests synchronous loading and invoking a tool with a dynamic callable secure parameter.""" + tool = toolbox.load_tool("my-secure-tool") + bound_tool = tool.bind_secure_param("name", lambda: "Alice") + response = bound_tool(id=1) + assert isinstance(response, str) + assert "Alice" in response + + def test_sync_load_tool_with_secure_params(self, toolbox: ToolboxSyncClient): + """Tests synchronous load_tool with secure_params passed during loading.""" + tool = toolbox.load_tool("my-secure-tool", secure_params={"name": "Alice"}) + response = tool(id=1) + assert isinstance(response, str) + assert "Alice" in response + + def test_sync_load_toolset_with_secure_params(self, toolbox: ToolboxSyncClient): + """Tests synchronous load_toolset with secure_params distributed across tools.""" + toolset = toolbox.load_toolset( + "my-secure-toolset", secure_params={"name": "Alice"} + ) + by_name = {t.__name__: t for t in toolset} + assert "my-secure-tool" in by_name + tool = by_name["my-secure-tool"] + response = tool(id=1) + assert isinstance(response, str) + assert "Alice" in response diff --git a/packages/toolbox-langchain/integration.cloudbuild.yaml b/packages/toolbox-langchain/integration.cloudbuild.yaml index ecacfd86b..9c2e0a3d0 100644 --- a/packages/toolbox-langchain/integration.cloudbuild.yaml +++ b/packages/toolbox-langchain/integration.cloudbuild.yaml @@ -50,4 +50,4 @@ options: substitutions: _VERSION: '3.13' _TOOLBOX_VERSION: '1.9.0' - _TOOLBOX_MANIFEST_VERSION: '34' + _TOOLBOX_MANIFEST_VERSION: '38' diff --git a/packages/toolbox-langchain/tests/test_e2e.py b/packages/toolbox-langchain/tests/test_e2e.py index 8dd095f48..3d7f8ba42 100644 --- a/packages/toolbox-langchain/tests/test_e2e.py +++ b/packages/toolbox-langchain/tests/test_e2e.py @@ -82,8 +82,7 @@ async def test_aload_toolset_specific( async def test_aload_toolset_all(self, toolbox): toolset = await toolbox.aload_toolset() - assert len(toolset) == 7 - tool_names = [ + expected_tools = [ "get-n-rows", "get-row-by-id", "get-row-by-id-auth", @@ -91,10 +90,12 @@ async def test_aload_toolset_all(self, toolbox): "get-row-by-content-auth", "search-rows", "process-data", + "my-secure-tool", ] - for tool in toolset: - name = tool._ToolboxTool__core_tool.__name__ - assert name in tool_names + assert len(toolset) == len(expected_tools) + assert {t._ToolboxTool__core_tool.__name__ for t in toolset} == set( + expected_tools + ) async def test_aload_toolset_explicit_protocol(self): toolbox = ToolboxClient( @@ -254,10 +255,9 @@ def test_load_toolset_specific( name = tool._ToolboxTool__core_tool.__name__ assert name in expected_tools - def test_aload_toolset_all(self, toolbox): + def test_load_toolset_all(self, toolbox): toolset = toolbox.load_toolset() - assert len(toolset) == 7 - tool_names = [ + expected_tools = [ "get-n-rows", "get-row-by-id", "get-row-by-id-auth", @@ -265,10 +265,12 @@ def test_aload_toolset_all(self, toolbox): "get-row-by-content-auth", "search-rows", "process-data", + "my-secure-tool", ] - for tool in toolset: - name = tool._ToolboxTool__core_tool.__name__ - assert name in tool_names + assert len(toolset) == len(expected_tools) + assert {t._ToolboxTool__core_tool.__name__ for t in toolset} == set( + expected_tools + ) def test_load_toolset_explicit_protocol(self): toolbox = ToolboxClient( @@ -392,3 +394,31 @@ def test_run_tool_param_auth_no_field(self, toolbox, auth_token1): 'provided parameters were invalid: error parsing authenticated parameter "data": no field named row_data in claims' in response ) + + +@pytest.mark.usefixtures("toolbox_server") +class TestSecureParamsE2E: + @pytest.mark.asyncio + async def test_async_run_tool_with_secure_param(self): + """Tests LangChain AsyncToolboxTool with secure parameters.""" + toolbox = ToolboxClient(TOOLBOX_SERVER_URL_STABLE) + try: + tool = await toolbox.aload_tool("my-secure-tool") + bound_tool = tool.bind_secure_param("name", "Alice") + response = await bound_tool.ainvoke({"id": 1}) + assert isinstance(response, str) + assert "Alice" in response + finally: + toolbox.close() + + def test_sync_run_tool_with_secure_param(self): + """Tests LangChain ToolboxTool with secure parameters.""" + toolbox = ToolboxClient(TOOLBOX_SERVER_URL_STABLE) + try: + tool = toolbox.load_tool("my-secure-tool") + bound_tool = tool.bind_secure_param("name", "Alice") + response = bound_tool.invoke({"id": 1}) + assert isinstance(response, str) + assert "Alice" in response + finally: + toolbox.close() diff --git a/packages/toolbox-llamaindex/integration.cloudbuild.yaml b/packages/toolbox-llamaindex/integration.cloudbuild.yaml index d96b51cfc..38d582a14 100644 --- a/packages/toolbox-llamaindex/integration.cloudbuild.yaml +++ b/packages/toolbox-llamaindex/integration.cloudbuild.yaml @@ -50,4 +50,4 @@ options: substitutions: _VERSION: '3.13' _TOOLBOX_VERSION: '1.9.0' - _TOOLBOX_MANIFEST_VERSION: '34' + _TOOLBOX_MANIFEST_VERSION: '38' diff --git a/packages/toolbox-llamaindex/tests/test_e2e.py b/packages/toolbox-llamaindex/tests/test_e2e.py index f8c93f701..6f0e0dc5a 100644 --- a/packages/toolbox-llamaindex/tests/test_e2e.py +++ b/packages/toolbox-llamaindex/tests/test_e2e.py @@ -82,8 +82,7 @@ async def test_aload_toolset_specific( async def test_aload_toolset_all(self, toolbox): toolset = await toolbox.aload_toolset() - assert len(toolset) == 7 - tool_names = [ + expected_tools = [ "get-n-rows", "get-row-by-id", "get-row-by-id-auth", @@ -91,10 +90,12 @@ async def test_aload_toolset_all(self, toolbox): "get-row-by-content-auth", "search-rows", "process-data", + "my-secure-tool", ] - for tool in toolset: - name = tool._ToolboxTool__core_tool.__name__ - assert name in tool_names + assert len(toolset) == len(expected_tools) + assert {t._ToolboxTool__core_tool.__name__ for t in toolset} == set( + expected_tools + ) async def test_aload_toolset_explicit_protocol(self): toolbox = ToolboxClient( @@ -254,10 +255,9 @@ def test_load_toolset_specific( name = tool._ToolboxTool__core_tool.__name__ assert name in expected_tools - def test_aload_toolset_all(self, toolbox): + def test_load_toolset_all(self, toolbox): toolset = toolbox.load_toolset() - assert len(toolset) == 7 - tool_names = [ + expected_tools = [ "get-n-rows", "get-row-by-id", "get-row-by-id-auth", @@ -265,10 +265,12 @@ def test_aload_toolset_all(self, toolbox): "get-row-by-content-auth", "search-rows", "process-data", + "my-secure-tool", ] - for tool in toolset: - name = tool._ToolboxTool__core_tool.__name__ - assert name in tool_names + assert len(toolset) == len(expected_tools) + assert {t._ToolboxTool__core_tool.__name__ for t in toolset} == set( + expected_tools + ) def test_load_toolset_explicit_protocol(self): toolbox = ToolboxClient( @@ -392,3 +394,31 @@ def test_run_tool_param_auth_no_field(self, toolbox, auth_token1): 'provided parameters were invalid: error parsing authenticated parameter "data": no field named row_data in claims' in response.content ) + + +@pytest.mark.usefixtures("toolbox_server") +class TestSecureParamsE2E: + @pytest.mark.asyncio + async def test_async_run_tool_with_secure_param(self): + """Tests LlamaIndex AsyncToolboxTool with secure parameters.""" + toolbox = ToolboxClient(TOOLBOX_SERVER_URL_STABLE) + try: + tool = await toolbox.aload_tool("my-secure-tool") + bound_tool = tool.bind_secure_param("name", "Alice") + response = await bound_tool.acall(id=1) + assert isinstance(response.content, str) + assert "Alice" in response.content + finally: + toolbox.close() + + def test_sync_run_tool_with_secure_param(self): + """Tests LlamaIndex ToolboxTool with secure parameters.""" + toolbox = ToolboxClient(TOOLBOX_SERVER_URL_STABLE) + try: + tool = toolbox.load_tool("my-secure-tool") + bound_tool = tool.bind_secure_param("name", "Alice") + response = bound_tool.call(id=1) + assert isinstance(response.content, str) + assert "Alice" in response.content + finally: + toolbox.close()