chore: create a utility to evaluate the mcp server implemenation, inspired by https://www.anthropic.com/engineering/writing-tools-for-agents

Author: bcallender

src/fenic/api/mcp/tool_generation.py

@@ -224,7 +224,6 @@ def _auto_generate_read_tool(
     name_to_df: Dict[str, DataFrame] = {d.table_name: d.df for d in datasets}
 
     async def _validate_columns(
-        ctx: Context,
         available_columns: List[str],
         original_columns: List[str],
         filtered_columns: List[str],
@@ -233,10 +232,9 @@ async def _validate_columns(
             raise ValidationError(f"Column(s) {original_columns} not found. Available: {', '.join(available_columns)}")
         if len(filtered_columns) != len(original_columns):
             invalid_columns = [c for c in original_columns if c not in filtered_columns]
-            await ctx.warning(f"Column(s) {invalid_columns} not found. Available: {', '.join(available_columns)}")
+            raise ValidationError(f"Column(s) {invalid_columns} not found. Available: {', '.join(available_columns)}")
 
     async def read_func(
-        ctx: Context, # MCP server context allows us to log warnings back to the client.
         df_name: Annotated[str, "Dataset name to read rows from."],
         limit: Annotated[Optional[int], "Max rows to read within a page"] = result_limit,
         offset: Annotated[Optional[int], "Row offset to start from (requires order_by)"] = None,
@@ -258,11 +256,11 @@ async def read_func(
         exclude_columns = [c.strip() for c in exclude_columns.split(",") if c.strip()] if exclude_columns else None
         if include_columns:
             filtered_columns = [c for c in include_columns if c in available_columns]
-            await _validate_columns(ctx, available_columns, include_columns, filtered_columns)
+            await _validate_columns(available_columns, include_columns, filtered_columns)
             df = df.select(*filtered_columns)
         if exclude_columns:
             filtered_columns = [c for c in available_columns if c not in exclude_columns]
-            await _validate_columns(ctx, available_columns, exclude_columns, filtered_columns)
+            await _validate_columns(available_columns, exclude_columns, filtered_columns)
             df = df.select(*filtered_columns)
         # Apply paging (handles offset+order_by via SQL and optional limit)
         return _apply_paging(

src/fenic/core/mcp/types.py

@@ -80,15 +80,15 @@ class ParameterizedToolDefinition:
 class DynamicToolDefinition:
     """A tool implemented as a regular Python function with explicit parameters.
 
-    The function must be a `Callable[..., Coroutine[Any, Any, LogicalPlan]]` 
+    The function must be a `Callable[..., Coroutine[Any, Any, LogicalPlan]]`
     (a function defined with `async def`). Collection/formatting is handled by
     the MCP generator wrapper.
     """
     name: str
     description: str
     max_result_limit: Optional[int]
     func: Callable[..., Coroutine[Any, Any, LogicalPlan]]
-    add_limit_parameter: bool = True 
+    add_limit_parameter: bool = True
     default_table_format: TableFormat = "markdown"
     read_only: Annotated[bool, "A hint to provide to the client that the tool is read-only."] = True
     idempotent: Annotated[bool, "A hint to provide to the client that the tool is idempotent."] = True