chore: emit strands metrics (#248)

Author: poshinchen

src/strands/event_loop/event_loop.py

@@ -88,11 +88,11 @@ def event_loop_cycle(
     kwargs["event_loop_cycle_id"] = uuid.uuid4()
 
     event_loop_metrics: EventLoopMetrics = kwargs.get("event_loop_metrics", EventLoopMetrics())
-
     # Initialize state and get cycle trace
     if "request_state" not in kwargs:
         kwargs["request_state"] = {}
-    cycle_start_time, cycle_trace = event_loop_metrics.start_cycle()
+    attributes = {"event_loop_cycle_id": str(kwargs.get("event_loop_cycle_id"))}
+    cycle_start_time, cycle_trace = event_loop_metrics.start_cycle(attributes=attributes)
     kwargs["event_loop_cycle_trace"] = cycle_trace
 
     callback_handler(start=True)
@@ -211,7 +211,7 @@ def event_loop_cycle(
             )
 
         # End the cycle and return results
-        event_loop_metrics.end_cycle(cycle_start_time, cycle_trace)
+        event_loop_metrics.end_cycle(cycle_start_time, cycle_trace, attributes)
         if cycle_span:
             tracer.end_event_loop_cycle_span(
                 span=cycle_span,
@@ -344,7 +344,6 @@ def _handle_tool_execution(
 
     if not tool_uses:
         return stop_reason, message, event_loop_metrics, kwargs["request_state"]
-
     tool_handler_process = partial(
         tool_handler.process,
         messages=messages,

src/strands/telemetry/metrics.py

@@ -7,7 +7,7 @@
 from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
 
 import opentelemetry.metrics as metrics_api
-from opentelemetry.metrics import Counter, Meter
+from opentelemetry.metrics import Counter, Histogram, Meter
 
 from ..telemetry import metrics_constants as constants
 from ..types.content import Message
@@ -121,22 +121,34 @@ class ToolMetrics:
     error_count: int = 0
     total_time: float = 0.0
 
-    def add_call(self, tool: ToolUse, duration: float, success: bool) -> None:
+    def add_call(
+        self,
+        tool: ToolUse,
+        duration: float,
+        success: bool,
+        metrics_client: "MetricsClient",
+        attributes: Optional[Dict[str, Any]] = None,
+    ) -> None:
         """Record a new tool call with its outcome.
 
         Args:
             tool: The tool that was called.
             duration: How long the call took in seconds.
             success: Whether the call was successful.
+            metrics_client: The metrics client for recording the metrics.
+            attributes: attributes of the metrics.
         """
         self.tool = tool  # Update with latest tool state
         self.call_count += 1
         self.total_time += duration
-
+        metrics_client.tool_call_count.add(1, attributes=attributes)
+        metrics_client.tool_duration.record(duration, attributes=attributes)
         if success:
             self.success_count += 1
+            metrics_client.tool_success_count.add(1, attributes=attributes)
         else:
             self.error_count += 1
+            metrics_client.tool_error_count.add(1, attributes=attributes)
 
 
 @dataclass
@@ -159,32 +171,53 @@ class EventLoopMetrics:
     accumulated_usage: Usage = field(default_factory=lambda: Usage(inputTokens=0, outputTokens=0, totalTokens=0))
     accumulated_metrics: Metrics = field(default_factory=lambda: Metrics(latencyMs=0))
 
-    def start_cycle(self) -> Tuple[float, Trace]:
+    @property
+    def _metrics_client(self) -> "MetricsClient":
+        """Get the singleton MetricsClient instance."""
+        return MetricsClient()
+
+    def start_cycle(
+        self,
+        attributes: Optional[Dict[str, Any]] = None,
+    ) -> Tuple[float, Trace]:
         """Start a new event loop cycle and create a trace for it.
 
+        Args:
+            attributes: attributes of the metrics.
+
         Returns:
             A tuple containing the start time and the cycle trace object.
         """
+        self._metrics_client.event_loop_cycle_count.add(1, attributes=attributes)
+        self._metrics_client.event_loop_start_cycle.add(1, attributes=attributes)
         self.cycle_count += 1
         start_time = time.time()
         cycle_trace = Trace(f"Cycle {self.cycle_count}", start_time=start_time)
         self.traces.append(cycle_trace)
         return start_time, cycle_trace
 
-    def end_cycle(self, start_time: float, cycle_trace: Trace) -> None:
+    def end_cycle(self, start_time: float, cycle_trace: Trace, attributes: Optional[Dict[str, Any]] = None) -> None:
         """End the current event loop cycle and record its duration.
 
         Args:
             start_time: The timestamp when the cycle started.
             cycle_trace: The trace object for this cycle.
+            attributes: attributes of the metrics.
         """
+        self._metrics_client.event_loop_end_cycle.add(1, attributes)
         end_time = time.time()
         duration = end_time - start_time
+        self._metrics_client.event_loop_cycle_duration.record(duration, attributes)
         self.cycle_durations.append(duration)
         cycle_trace.end(end_time)
 
     def add_tool_usage(
-        self, tool: ToolUse, duration: float, tool_trace: Trace, success: bool, message: Message
+        self,
+        tool: ToolUse,
+        duration: float,
+        tool_trace: Trace,
+        success: bool,
+        message: Message,
     ) -> None:
         """Record metrics for a tool invocation.
 
@@ -207,8 +240,16 @@ def add_tool_usage(
         tool_trace.raw_name = f"{tool_name} - {tool_use_id}"
         tool_trace.add_message(message)
 
-        self.tool_metrics.setdefault(tool_name, ToolMetrics(tool)).add_call(tool, duration, success)
-
+        self.tool_metrics.setdefault(tool_name, ToolMetrics(tool)).add_call(
+            tool,
+            duration,
+            success,
+            self._metrics_client,
+            attributes={
+                "tool_name": tool_name,
+                "tool_use_id": tool_use_id,
+            },
+        )
         tool_trace.end()
 
     def update_usage(self, usage: Usage) -> None:
@@ -217,6 +258,8 @@ def update_usage(self, usage: Usage) -> None:
         Args:
             usage: The usage data to add to the accumulated totals.
         """
+        self._metrics_client.event_loop_input_tokens.record(usage["inputTokens"])
+        self._metrics_client.event_loop_output_tokens.record(usage["outputTokens"])
         self.accumulated_usage["inputTokens"] += usage["inputTokens"]
         self.accumulated_usage["outputTokens"] += usage["outputTokens"]
         self.accumulated_usage["totalTokens"] += usage["totalTokens"]
@@ -227,6 +270,7 @@ def update_metrics(self, metrics: Metrics) -> None:
         Args:
             metrics: The metrics data to add to the accumulated totals.
         """
+        self._metrics_client.event_loop_latency.record(metrics["latencyMs"])
         self.accumulated_metrics["latencyMs"] += metrics["latencyMs"]
 
     def get_summary(self) -> Dict[str, Any]:
@@ -370,7 +414,18 @@ class MetricsClient:
 
     _instance: Optional["MetricsClient"] = None
     meter: Meter
-    strands_agent_invocation_count: Counter
+    event_loop_cycle_count: Counter
+    event_loop_start_cycle: Counter
+    event_loop_end_cycle: Counter
+    event_loop_cycle_duration: Histogram
+    event_loop_latency: Histogram
+    event_loop_input_tokens: Histogram
+    event_loop_output_tokens: Histogram
+
+    tool_call_count: Counter
+    tool_success_count: Counter
+    tool_error_count: Counter
+    tool_duration: Histogram
 
     def __new__(cls) -> "MetricsClient":
         """Create or return the singleton instance of MetricsClient.
@@ -398,6 +453,24 @@ def __init__(self) -> None:
 
     def create_instruments(self) -> None:
         """Create and initialize all OpenTelemetry metric instruments."""
-        self.strands_agent_invocation_count = self.meter.create_counter(
-            name=constants.STRANDS_AGENT_INVOCATION_COUNT, unit="Count"
+        self.event_loop_cycle_count = self.meter.create_counter(
+            name=constants.STRANDS_EVENT_LOOP_CYCLE_COUNT, unit="Count"
+        )
+        self.event_loop_start_cycle = self.meter.create_counter(
+            name=constants.STRANDS_EVENT_LOOP_START_CYCLE, unit="Count"
+        )
+        self.event_loop_end_cycle = self.meter.create_counter(name=constants.STRANDS_EVENT_LOOP_END_CYCLE, unit="Count")
+        self.event_loop_cycle_duration = self.meter.create_histogram(
+            name=constants.STRANDS_EVENT_LOOP_CYCLE_DURATION, unit="s"
+        )
+        self.event_loop_latency = self.meter.create_histogram(name=constants.STRANDS_EVENT_LOOP_LATENCY, unit="ms")
+        self.tool_call_count = self.meter.create_counter(name=constants.STRANDS_TOOL_CALL_COUNT, unit="Count")
+        self.tool_success_count = self.meter.create_counter(name=constants.STRANDS_TOOL_SUCCESS_COUNT, unit="Count")
+        self.tool_error_count = self.meter.create_counter(name=constants.STRANDS_TOOL_ERROR_COUNT, unit="Count")
+        self.tool_duration = self.meter.create_histogram(name=constants.STRANDS_TOOL_DURATION, unit="s")
+        self.event_loop_input_tokens = self.meter.create_histogram(
+            name=constants.STRANDS_EVENT_LOOP_INPUT_TOKENS, unit="token"
+        )
+        self.event_loop_output_tokens = self.meter.create_histogram(
+            name=constants.STRANDS_EVENT_LOOP_OUTPUT_TOKENS, unit="token"
         )

src/strands/telemetry/metrics_constants.py

@@ -1,3 +1,15 @@
-"""Metrics that are emitted in Strands-Agent."""
+"""Metrics that are emitted in Strands-Agents."""
 
-STRANDS_AGENT_INVOCATION_COUNT = "strands.agent.invocation_count"
+STRANDS_EVENT_LOOP_CYCLE_COUNT = "strands.event_loop.cycle_count"
+STRANDS_EVENT_LOOP_START_CYCLE = "strands.event_loop.start_cycle"
+STRANDS_EVENT_LOOP_END_CYCLE = "strands.event_loop.end_cycle"
+STRANDS_TOOL_CALL_COUNT = "strands.tool.call_count"
+STRANDS_TOOL_SUCCESS_COUNT = "strands.tool.success_count"
+STRANDS_TOOL_ERROR_COUNT = "strands.tool.error_count"
+
+# Histograms
+STRANDS_EVENT_LOOP_LATENCY = "strands.event_loop.latency"
+STRANDS_TOOL_DURATION = "strands.tool.duration"
+STRANDS_EVENT_LOOP_CYCLE_DURATION = "strands.event_loop.cycle_duration"
+STRANDS_EVENT_LOOP_INPUT_TOKENS = "strands.event_loop.input.tokens"
+STRANDS_EVENT_LOOP_OUTPUT_TOKENS = "strands.event_loop.output.tokens"

tests/strands/telemetry/test_metrics.py

@@ -124,12 +124,19 @@ def test_trace_end(mock_time, end_time, trace):
 @pytest.fixture
 def mock_get_meter_provider():
     with mock.patch("strands.telemetry.metrics.metrics_api.get_meter_provider") as mock_get_meter_provider:
+        MetricsClient._instance = None
         meter_provider_mock = mock.MagicMock(spec=MeterProvider)
-        mock_get_meter_provider.return_value = meter_provider_mock
 
         mock_meter = mock.MagicMock()
+        mock_create_counter = mock.MagicMock()
+        mock_meter.create_counter.return_value = mock_create_counter
+
+        mock_create_histogram = mock.MagicMock()
+        mock_meter.create_histogram.return_value = mock_create_histogram
         meter_provider_mock.get_meter.return_value = mock_meter
 
+        mock_get_meter_provider.return_value = meter_provider_mock
+
         yield mock_get_meter_provider
 
 
@@ -190,11 +197,14 @@ def test_trace_to_dict(trace):
 
 
 @pytest.mark.parametrize("success", [True, False])
-def test_tool_metrics_add_call(success, tool, tool_metrics):
+def test_tool_metrics_add_call(success, tool, tool_metrics, mock_get_meter_provider):
     tool = dict(tool, **{"name": "updated"})
     duration = 1
+    metrics_client = MetricsClient()
+
+    attributes = {"foo": "bar"}
 
-    tool_metrics.add_call(tool, duration, success)
+    tool_metrics.add_call(tool, duration, success, metrics_client, attributes=attributes)
 
     tru_attrs = dataclasses.asdict(tool_metrics)
     exp_attrs = {
@@ -205,12 +215,17 @@ def test_tool_metrics_add_call(success, tool, tool_metrics):
         "total_time": duration,
     }
 
+    mock_get_meter_provider.return_value.get_meter.assert_called()
+    metrics_client.tool_call_count.add.assert_called_with(1, attributes=attributes)
+    metrics_client.tool_duration.record.assert_called_with(duration, attributes=attributes)
+    if success:
+        metrics_client.tool_success_count.add.assert_called_with(1, attributes=attributes)
     assert tru_attrs == exp_attrs
 
 
 @unittest.mock.patch.object(strands.telemetry.metrics.time, "time")
 @unittest.mock.patch.object(strands.telemetry.metrics.uuid, "uuid4")
-def test_event_loop_metrics_start_cycle(mock_uuid4, mock_time, event_loop_metrics):
+def test_event_loop_metrics_start_cycle(mock_uuid4, mock_time, event_loop_metrics, mock_get_meter_provider):
     mock_time.return_value = 1
     mock_uuid4.return_value = "i1"
 
@@ -220,6 +235,8 @@ def test_event_loop_metrics_start_cycle(mock_uuid4, mock_time, event_loop_metric
     tru_attrs = {"cycle_count": event_loop_metrics.cycle_count, "traces": event_loop_metrics.traces}
     exp_attrs = {"cycle_count": 1, "traces": [tru_cycle_trace]}
 
+    mock_get_meter_provider.return_value.get_meter.assert_called()
+    event_loop_metrics._metrics_client.event_loop_cycle_count.add.assert_called()
     assert (
         tru_start_time == exp_start_time
         and tru_cycle_trace.to_dict() == exp_cycle_trace.to_dict()
@@ -228,10 +245,11 @@ def test_event_loop_metrics_start_cycle(mock_uuid4, mock_time, event_loop_metric
 
 
 @unittest.mock.patch.object(strands.telemetry.metrics.time, "time")
-def test_event_loop_metrics_end_cycle(mock_time, trace, event_loop_metrics):
+def test_event_loop_metrics_end_cycle(mock_time, trace, event_loop_metrics, mock_get_meter_provider):
     mock_time.return_value = 1
 
-    event_loop_metrics.end_cycle(start_time=0, cycle_trace=trace)
+    attributes = {"foo": "bar"}
+    event_loop_metrics.end_cycle(start_time=0, cycle_trace=trace, attributes=attributes)
 
     tru_cycle_durations = event_loop_metrics.cycle_durations
     exp_cycle_durations = [1]
@@ -243,17 +261,23 @@ def test_event_loop_metrics_end_cycle(mock_time, trace, event_loop_metrics):
 
     assert tru_trace_end_time == exp_trace_end_time
 
+    mock_get_meter_provider.return_value.get_meter.assert_called()
+    metrics_client = event_loop_metrics._metrics_client
+    metrics_client.event_loop_end_cycle.add.assert_called_with(1, attributes)
+    metrics_client.event_loop_cycle_duration.record.assert_called()
+
 
 @unittest.mock.patch.object(strands.telemetry.metrics.time, "time")
-def test_event_loop_metrics_add_tool_usage(mock_time, trace, tool, event_loop_metrics):
+def test_event_loop_metrics_add_tool_usage(mock_time, trace, tool, event_loop_metrics, mock_get_meter_provider):
     mock_time.return_value = 1
-
     duration = 1
     success = True
     message = {"role": "user", "content": [{"toolResult": {"toolUseId": "123", "tool_name": "tool1"}}]}
 
     event_loop_metrics.add_tool_usage(tool, duration, trace, success, message)
 
+    mock_get_meter_provider.return_value.get_meter.assert_called()
+
     tru_event_loop_metrics_attrs = {"tool_metrics": event_loop_metrics.tool_metrics}
     exp_event_loop_metrics_attrs = {
         "tool_metrics": {
@@ -286,7 +310,7 @@ def test_event_loop_metrics_add_tool_usage(mock_time, trace, tool, event_loop_me
     assert tru_trace_attrs == exp_trace_attrs
 
 
-def test_event_loop_metrics_update_usage(usage, event_loop_metrics):
+def test_event_loop_metrics_update_usage(usage, event_loop_metrics, mock_get_meter_provider):
     for _ in range(3):
         event_loop_metrics.update_usage(usage)
 
@@ -298,9 +322,13 @@ def test_event_loop_metrics_update_usage(usage, event_loop_metrics):
     )
 
     assert tru_usage == exp_usage
+    mock_get_meter_provider.return_value.get_meter.assert_called()
+    metrics_client = event_loop_metrics._metrics_client
+    metrics_client.event_loop_input_tokens.record.assert_called()
+    metrics_client.event_loop_output_tokens.record.assert_called()
 
 
-def test_event_loop_metrics_update_metrics(metrics, event_loop_metrics):
+def test_event_loop_metrics_update_metrics(metrics, event_loop_metrics, mock_get_meter_provider):
     for _ in range(3):
         event_loop_metrics.update_metrics(metrics)
 
@@ -310,9 +338,11 @@ def test_event_loop_metrics_update_metrics(metrics, event_loop_metrics):
     )
 
     assert tru_metrics == exp_metrics
+    mock_get_meter_provider.return_value.get_meter.assert_called()
+    event_loop_metrics._metrics_client.event_loop_latency.record.assert_called_with(1)
 
 
-def test_event_loop_metrics_get_summary(trace, tool, event_loop_metrics):
+def test_event_loop_metrics_get_summary(trace, tool, event_loop_metrics, mock_get_meter_provider):
     duration = 1
     success = True
     message = {"role": "user", "content": [{"toolResult": {"toolUseId": "123", "tool_name": "tool1"}}]}

tests/strands/tools/test_executor.py

@@ -43,6 +43,12 @@ def tool_uses(request, tool_use):
     return request.param if hasattr(request, "param") else [tool_use]
 
 
+@pytest.fixture
+def mock_metrics_client():
+    with unittest.mock.patch("strands.telemetry.MetricsClient") as mock_metrics_client:
+        yield mock_metrics_client
+
+
 @pytest.fixture
 def event_loop_metrics():
     return strands.telemetry.metrics.EventLoopMetrics()
@@ -303,6 +309,7 @@ def test_run_tools_creates_and_ends_span_on_success(
     mock_get_tracer,
     tool_handler,
     tool_uses,
+    mock_metrics_client,
     event_loop_metrics,
     request_state,
     invalid_tool_use_ids,