From dffabb048c6f05cfde1564b547338c4bc34b4f7a Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 20 Oct 2025 18:49:00 -0400 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=A4=96=20Add=20image=20reading=20supp?= =?UTF-8?q?ort=20to=20file=5Fread=20tool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Detect MIME types using mime-types package - Read images as base64 binary data - Use toModelOutput to send images as media content to AI models - Update UI to display image previews - Add comprehensive tests for image reading and conversion --- bun.lock | 86 +++++------ package.json | 71 ++++----- src/components/tools/FileReadToolCall.tsx | 46 ++++-- src/services/tools/file_read.test.ts | 176 ++++++++++++++++++++++ src/services/tools/file_read.ts | 42 +++++- src/types/tools.ts | 1 + 6 files changed, 313 insertions(+), 109 deletions(-) diff --git a/bun.lock b/bun.lock index d72e1a4ec..7c86d7dbf 100644 --- a/bun.lock +++ b/bun.lock @@ -6,30 +6,44 @@ "dependencies": { "@ai-sdk/anthropic": "^2.0.29", "@ai-sdk/openai": "^2.0.52", + "@emotion/react": "^11.14.0", + "@emotion/styled": "^11.14.1", "ai": "^5.0.72", "ai-tokenizer": "^1.0.3", "chalk": "^5.6.2", - "cors": "^2.8.5", + "cmdk": "^1.0.0", "crc-32": "^1.2.2", "diff": "^8.0.2", "disposablestack": "^1.1.7", - "electron": "^38.2.1", "electron-updater": "^6.6.2", - "express": "^5.1.0", + "escape-html": "^1.0.3", "jsonc-parser": "^3.3.1", "lru-cache": "^11.2.2", + "markdown-it": "^14.1.0", + "mermaid": "^11.12.0", + "mime-types": "^3.0.1", "minimist": "^1.2.8", + "posthog-js": "^1.276.0", + "react": "^18.2.0", + "react-compiler-runtime": "^1.0.0", + "react-dnd": "^16.0.1", + "react-dnd-html5-backend": "^16.0.1", + "react-dom": "^18.2.0", + "react-markdown": "^10.1.0", + "rehype-katex": "^7.0.1", + "rehype-raw": "^7.0.0", + "rehype-sanitize": "^6.0.0", + "remark-gfm": "^4.0.1", + "remark-math": "^6.0.0", + "shiki": "^3.13.0", "source-map-support": "^0.5.21", "undici": "^7.16.0", "write-file-atomic": "^6.0.0", - "ws": "^8.18.3", "zod": "^4.1.11", "zod-to-json-schema": "^3.24.6", }, "devDependencies": { "@emotion/babel-plugin": "^11.13.5", - "@emotion/react": "^11.14.0", - "@emotion/styled": "^11.14.1", "@eslint/js": "^9.36.0", "@playwright/test": "^1.56.0", "@storybook/addon-essentials": "^8.6.14", @@ -48,6 +62,7 @@ "@types/jest": "^30.0.0", "@types/katex": "^0.16.7", "@types/markdown-it": "^14.1.2", + "@types/mime-types": "^3.0.1", "@types/minimist": "^1.2.5", "@types/react": "^18.2.0", "@types/react-dom": "^18.2.0", @@ -58,34 +73,20 @@ "@typescript/native-preview": "^7.0.0-dev.20251014.1", "@vitejs/plugin-react": "^4.0.0", "babel-plugin-react-compiler": "^1.0.0", - "cmdk": "^1.0.0", "concurrently": "^8.2.0", + "cors": "^2.8.5", "dotenv": "^17.2.3", + "electron": "^38.2.1", "electron-builder": "^24.6.0", "electron-devtools-installer": "^4.0.0", "electron-mock-ipc": "^0.3.12", - "escape-html": "^1.0.3", "eslint": "^9.36.0", "eslint-plugin-react": "^7.37.5", "eslint-plugin-react-hooks": "^5.2.0", + "express": "^5.1.0", "jest": "^30.1.3", - "markdown-it": "^14.1.0", - "mermaid": "^11.12.0", "playwright": "^1.56.0", - "posthog-js": "^1.276.0", "prettier": "^3.6.2", - "react": "^18.2.0", - "react-compiler-runtime": "^1.0.0", - "react-dnd": "^16.0.1", - "react-dnd-html5-backend": "^16.0.1", - "react-dom": "^18.2.0", - "react-markdown": "^10.1.0", - "rehype-katex": "^7.0.1", - "rehype-raw": "^7.0.0", - "rehype-sanitize": "^6.0.0", - "remark-gfm": "^4.0.1", - "remark-math": "^6.0.0", - "shiki": "^3.13.0", "storybook": "^8.6.14", "ts-jest": "^29.4.4", "tsc-alias": "^1.8.16", @@ -94,6 +95,7 @@ "vite": "^4.4.0", "vite-plugin-svgr": "^4.5.0", "vite-plugin-top-level-await": "^1.6.0", + "ws": "^8.18.3", }, }, }, @@ -744,6 +746,8 @@ "@types/mime": ["@types/mime@1.3.5", "", {}, "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w=="], + "@types/mime-types": ["@types/mime-types@3.0.1", "", {}, "sha512-xRMsfuQbnRq1Ef+C+RKaENOxXX87Ygl38W1vDfPHRku02TgQr+Qd8iivLtAMcR0KF5/29xlnFihkTlbqFrGOVQ=="], + "@types/minimist": ["@types/minimist@1.2.5", "", {}, "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag=="], "@types/ms": ["@types/ms@2.1.0", "", {}, "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA=="], @@ -2294,7 +2298,7 @@ "react-dom": ["react-dom@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" }, "peerDependencies": { "react": "^18.3.1" } }, "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw=="], - "react-is": ["react-is@16.13.1", "", {}, "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="], + "react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], "react-markdown": ["react-markdown@10.1.0", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "hast-util-to-jsx-runtime": "^2.0.0", "html-url-attributes": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.0.0", "unified": "^11.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" }, "peerDependencies": { "@types/react": ">=18", "react": ">=18" } }, "sha512-qKxVopLT/TyA6BX3Ue5NwabOsAzm0Q7kAPwq6L+wWDwisYs7R8vZ0nRXqq6rkueboxpkjvLGU9fWifiX/ZZFxQ=="], @@ -2656,7 +2660,7 @@ "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], - "uuid": ["uuid@11.1.0", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A=="], + "uuid": ["uuid@10.0.0", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ=="], "v8-to-istanbul": ["v8-to-istanbul@9.3.0", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.12", "@types/istanbul-lib-coverage": "^2.0.1", "convert-source-map": "^2.0.0" } }, "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA=="], @@ -3008,6 +3012,8 @@ "hast-util-to-parse5/property-information": ["property-information@6.5.0", "", {}, "sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig=="], + "hoist-non-react-statics/react-is": ["react-is@16.13.1", "", {}, "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="], + "hosted-git-info/lru-cache": ["lru-cache@6.0.0", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA=="], "htmlparser2/entities": ["entities@1.1.2", "", {}, "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="], @@ -3142,6 +3148,8 @@ "mermaid/stylis": ["stylis@4.3.6", "", {}, "sha512-yQ3rwFWRfwNUY7H5vpU0wfdkNSnvnJinhF9830Swlaxl03zsOjCfmX0ugac+3LtK0lYSgwL/KXc8oYL3mG4YFQ=="], + "mermaid/uuid": ["uuid@11.1.0", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A=="], + "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], "minizlib/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="], @@ -3166,7 +3174,7 @@ "pretty-format/@jest/schemas": ["@jest/schemas@30.0.5", "", { "dependencies": { "@sinclair/typebox": "^0.34.0" } }, "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA=="], - "pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], + "prop-types/react-is": ["react-is@16.13.1", "", {}, "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="], "raw-body/iconv-lite": ["iconv-lite@0.7.0", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ=="], @@ -3222,8 +3230,6 @@ "vite/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], - "vite-plugin-top-level-await/uuid": ["uuid@10.0.0", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ=="], - "wait-port/chalk": ["chalk@2.4.2", "", { "dependencies": { "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", "supports-color": "^5.3.0" } }, "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ=="], "wait-port/commander": ["commander@3.0.2", "", {}, "sha512-Gar0ASD4BDyKC4hl4DwHqDrmvjoxWKZigVnAbn5H1owvm4CxCPdb0HQDehwNYMJpla5+M2tPmPARzhtYuwpHow=="], @@ -3488,8 +3494,6 @@ "jest-circus/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], - "jest-circus/pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - "jest-cli/@jest/test-result/@jest/console": ["@jest/console@30.2.0", "", { "dependencies": { "@jest/types": "30.2.0", "@types/node": "*", "chalk": "^4.1.2", "jest-message-util": "30.2.0", "jest-util": "30.2.0", "slash": "^3.0.0" } }, "sha512-+O1ifRjkvYIkBqASKWgLxrpEhQAAE7hY77ALLUufSk5717KfOShg6IbqLmdsLMPdUiFvA2kTs0R7YZy+l0IzZQ=="], "jest-cli/@jest/types/@jest/schemas": ["@jest/schemas@30.0.5", "", { "dependencies": { "@sinclair/typebox": "^0.34.0" } }, "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA=="], @@ -3558,28 +3562,18 @@ "jest-diff/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], - "jest-diff/pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - "jest-each/chalk/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], "jest-each/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], - "jest-each/pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - - "jest-leak-detector/pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - "jest-matcher-utils/chalk/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], "jest-matcher-utils/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], - "jest-matcher-utils/pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - "jest-message-util/chalk/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], "jest-message-util/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], - "jest-message-util/pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - "jest-process-manager/chalk/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], "jest-process-manager/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], @@ -3620,8 +3614,6 @@ "jest-snapshot/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], - "jest-snapshot/pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - "jest-util/chalk/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], "jest-util/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], @@ -3760,8 +3752,6 @@ "create-jest/jest-config/babel-jest/babel-preset-jest": ["babel-preset-jest@29.6.3", "", { "dependencies": { "babel-plugin-jest-hoist": "^29.6.3", "babel-preset-current-node-syntax": "^1.0.0" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA=="], - "create-jest/jest-config/pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - "expect/jest-matcher-utils/chalk/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], "expect/jest-matcher-utils/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], @@ -3880,8 +3870,6 @@ "jest-resolve-dependencies/jest-snapshot/jest-util/ci-info": ["ci-info@4.3.1", "", {}, "sha512-Wdy2Igu8OcBpI2pZePZ5oWjPC38tmDVx5WKUXKwlLYkA0ozo85sLsLvkBbBn/sZaSCMFOGZJ14fvW9t5/d7kdA=="], - "jest-resolve/jest-validate/pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - "jest-validate/@jest/types/@jest/schemas/@sinclair/typebox": ["@sinclair/typebox@0.34.41", "", {}, "sha512-6gS8pZzSXdyRHTIqoqSVknxolr1kzfy4/CeDnrzsVz8TTIWUbOBr6gnzOmTYJ3eXQNh4IYHIGi5aIL7sOZ2G/g=="], "jest/@jest/types/@jest/schemas/@sinclair/typebox": ["@sinclair/typebox@0.34.41", "", {}, "sha512-6gS8pZzSXdyRHTIqoqSVknxolr1kzfy4/CeDnrzsVz8TTIWUbOBr6gnzOmTYJ3eXQNh4IYHIGi5aIL7sOZ2G/g=="], @@ -3950,8 +3938,6 @@ "@storybook/test-runner/jest/@jest/core/jest-config/glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="], - "@storybook/test-runner/jest/@jest/core/pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - "@storybook/test-runner/jest/jest-cli/chalk/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], "@storybook/test-runner/jest/jest-cli/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], @@ -4046,10 +4032,6 @@ "@storybook/test-runner/jest/jest-cli/jest-config/babel-jest/babel-preset-jest": ["babel-preset-jest@29.6.3", "", { "dependencies": { "babel-plugin-jest-hoist": "^29.6.3", "babel-preset-current-node-syntax": "^1.0.0" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA=="], - "@storybook/test-runner/jest/jest-cli/jest-config/pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - - "@storybook/test-runner/jest/jest-cli/jest-validate/pretty-format/react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - "jest-config/jest-circus/jest-runtime/@jest/transform/babel-plugin-istanbul/istanbul-lib-instrument": ["istanbul-lib-instrument@6.0.3", "", { "dependencies": { "@babel/core": "^7.23.9", "@babel/parser": "^7.23.9", "@istanbuljs/schema": "^0.1.3", "istanbul-lib-coverage": "^3.2.0", "semver": "^7.5.4" } }, "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q=="], "jest-config/jest-circus/jest-snapshot/@jest/transform/babel-plugin-istanbul/istanbul-lib-instrument": ["istanbul-lib-instrument@6.0.3", "", { "dependencies": { "@babel/core": "^7.23.9", "@babel/parser": "^7.23.9", "@istanbuljs/schema": "^0.1.3", "istanbul-lib-coverage": "^3.2.0", "semver": "^7.5.4" } }, "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q=="], diff --git a/package.json b/package.json index 26092b81e..881e89d2a 100644 --- a/package.json +++ b/package.json @@ -1,20 +1,9 @@ { - "name": "@coder/cmux", + "name": "cmux", "version": "0.3.0", "description": "cmux - coder multiplexer", - "author": "Coder", "main": "dist/main.js", - "bin": { - "cmux": "dist/main.js" - }, "license": "AGPL-3.0-only", - "repository": { - "type": "git", - "url": "https://github.com/coder/cmux.git" - }, - "publishConfig": { - "access": "public" - }, "scripts": { "dev": "make dev", "prebuild:main": "./scripts/generate-version.sh", @@ -46,29 +35,44 @@ "dependencies": { "@ai-sdk/anthropic": "^2.0.29", "@ai-sdk/openai": "^2.0.52", + "@emotion/react": "^11.14.0", + "@emotion/styled": "^11.14.1", "ai": "^5.0.72", "ai-tokenizer": "^1.0.3", "chalk": "^5.6.2", - "cors": "^2.8.5", + "cmdk": "^1.0.0", "crc-32": "^1.2.2", "diff": "^8.0.2", "disposablestack": "^1.1.7", "electron-updater": "^6.6.2", - "express": "^5.1.0", + "escape-html": "^1.0.3", "jsonc-parser": "^3.3.1", "lru-cache": "^11.2.2", + "markdown-it": "^14.1.0", + "mermaid": "^11.12.0", + "mime-types": "^3.0.1", "minimist": "^1.2.8", + "posthog-js": "^1.276.0", + "react": "^18.2.0", + "react-compiler-runtime": "^1.0.0", + "react-dnd": "^16.0.1", + "react-dnd-html5-backend": "^16.0.1", + "react-dom": "^18.2.0", + "react-markdown": "^10.1.0", + "rehype-katex": "^7.0.1", + "rehype-raw": "^7.0.0", + "rehype-sanitize": "^6.0.0", + "remark-gfm": "^4.0.1", + "remark-math": "^6.0.0", + "shiki": "^3.13.0", "source-map-support": "^0.5.21", "undici": "^7.16.0", "write-file-atomic": "^6.0.0", - "ws": "^8.18.3", "zod": "^4.1.11", "zod-to-json-schema": "^3.24.6" }, "devDependencies": { "@emotion/babel-plugin": "^11.13.5", - "@emotion/react": "^11.14.0", - "@emotion/styled": "^11.14.1", "@eslint/js": "^9.36.0", "@playwright/test": "^1.56.0", "@storybook/addon-essentials": "^8.6.14", @@ -87,6 +91,7 @@ "@types/jest": "^30.0.0", "@types/katex": "^0.16.7", "@types/markdown-it": "^14.1.2", + "@types/mime-types": "^3.0.1", "@types/minimist": "^1.2.5", "@types/react": "^18.2.0", "@types/react-dom": "^18.2.0", @@ -97,8 +102,8 @@ "@typescript/native-preview": "^7.0.0-dev.20251014.1", "@vitejs/plugin-react": "^4.0.0", "babel-plugin-react-compiler": "^1.0.0", - "cmdk": "^1.0.0", "concurrently": "^8.2.0", + "cors": "^2.8.5", "dotenv": "^17.2.3", "electron": "^38.2.1", "electron-builder": "^24.6.0", @@ -107,25 +112,10 @@ "eslint": "^9.36.0", "eslint-plugin-react": "^7.37.5", "eslint-plugin-react-hooks": "^5.2.0", - "escape-html": "^1.0.3", + "express": "^5.1.0", "jest": "^30.1.3", - "markdown-it": "^14.1.0", - "mermaid": "^11.12.0", "playwright": "^1.56.0", - "posthog-js": "^1.276.0", "prettier": "^3.6.2", - "react": "^18.2.0", - "react-compiler-runtime": "^1.0.0", - "react-dnd": "^16.0.1", - "react-dnd-html5-backend": "^16.0.1", - "react-dom": "^18.2.0", - "react-markdown": "^10.1.0", - "rehype-katex": "^7.0.1", - "rehype-raw": "^7.0.0", - "rehype-sanitize": "^6.0.0", - "remark-gfm": "^4.0.1", - "remark-math": "^6.0.0", - "shiki": "^3.13.0", "storybook": "^8.6.14", "ts-jest": "^29.4.4", "tsc-alias": "^1.8.16", @@ -133,18 +123,9 @@ "typescript-eslint": "^8.45.0", "vite": "^4.4.0", "vite-plugin-svgr": "^4.5.0", - "vite-plugin-top-level-await": "^1.6.0" + "vite-plugin-top-level-await": "^1.6.0", + "ws": "^8.18.3" }, - "files": [ - "dist/**/*.js", - "dist/**/*.js.map", - "dist/**/*.wasm", - "dist/**/*.html", - "dist/**/*.css", - "dist/assets/**/*", - "README.md", - "LICENSE" - ], "build": { "appId": "com.cmux.app", "productName": "cmux", diff --git a/src/components/tools/FileReadToolCall.tsx b/src/components/tools/FileReadToolCall.tsx index 3e3a99cde..2a676a87e 100644 --- a/src/components/tools/FileReadToolCall.tsx +++ b/src/components/tools/FileReadToolCall.tsx @@ -101,6 +101,14 @@ const InfoValue = styled.span` word-break: break-all; `; +const ImagePreview = styled.img` + max-width: 100%; + max-height: 400px; + border-radius: 3px; + display: block; + margin: 8px 0; +`; + interface FileReadToolCallProps { args: FileReadToolArgs; result?: FileReadToolResult; @@ -170,7 +178,10 @@ export const FileReadToolCall: React.FC = ({ file_read {filePath} - {result && result.success && parsedContent && ( + {result && result.success && result.mime_type?.startsWith("image/") && ( + {result.mime_type} + )} + {result && result.success && parsedContent && !result.mime_type?.startsWith("image/") && ( read {formatBytes(parsedContent.actualBytes)} of {formatBytes(result.file_size)} @@ -210,19 +221,32 @@ export const FileReadToolCall: React.FC = ({ )} - {result.success && result.content && parsedContent && ( + {result.success && result.mime_type?.startsWith("image/") && ( - Content - - - {parsedContent.lineNumbers.map((lineNum, i) => ( -
{lineNum}
- ))} - - {parsedContent.actualContent} - + Image Preview + )} + + {result.success && + result.content && + !result.mime_type?.startsWith("image/") && + parsedContent && ( + + Content + + + {parsedContent.lineNumbers.map((lineNum, i) => ( +
{lineNum}
+ ))} + + {parsedContent.actualContent} + + + )} )} diff --git a/src/services/tools/file_read.test.ts b/src/services/tools/file_read.test.ts index 61129c85a..c4dd26b8c 100644 --- a/src/services/tools/file_read.test.ts +++ b/src/services/tools/file_read.test.ts @@ -388,4 +388,180 @@ describe("file_read tool", () => { expect(result.content).toContain("content in subdir"); } }); + + it("should read image files and return base64 content with mime type", async () => { + // Setup - create a simple 1x1 PNG image (smallest valid PNG) + const pngBuffer = Buffer.from([ + 0x89, + 0x50, + 0x4e, + 0x47, + 0x0d, + 0x0a, + 0x1a, + 0x0a, // PNG signature + 0x00, + 0x00, + 0x00, + 0x0d, + 0x49, + 0x48, + 0x44, + 0x52, // IHDR chunk + 0x00, + 0x00, + 0x00, + 0x01, + 0x00, + 0x00, + 0x00, + 0x01, // 1x1 dimensions + 0x08, + 0x06, + 0x00, + 0x00, + 0x00, + 0x1f, + 0x15, + 0xc4, + 0x89, + 0x00, + 0x00, + 0x00, + 0x0a, + 0x49, + 0x44, + 0x41, + 0x54, + 0x78, + 0x9c, + 0x63, + 0x00, + 0x01, + 0x00, + 0x00, + 0x05, + 0x00, + 0x01, + 0x0d, + 0x0a, + 0x2d, + 0xb4, + 0x00, + 0x00, + 0x00, + 0x00, + 0x49, + 0x45, + 0x4e, + 0x44, + 0xae, + 0x42, + 0x60, + 0x82, + ]); + const imagePath = path.join(testDir, "test.png"); + await fs.writeFile(imagePath, pngBuffer); + + using testEnv = createTestFileReadTool({ cwd: testDir }); + const tool = testEnv.tool; + const args: FileReadToolArgs = { + filePath: imagePath, + }; + + // Execute + const result = (await tool.execute!(args, mockToolCallOptions)) as FileReadToolResult; + + // Assert + expect(result.success).toBe(true); + if (result.success) { + expect(result.mime_type).toBe("image/png"); + expect(result.lines_read).toBe(0); // Images don't have lines + expect(result.content).toBe(pngBuffer.toString("base64")); + expect(result.file_size).toBe(pngBuffer.length); + } + }); + + it("should return media content for images via toModelOutput", async () => { + // Setup - create a simple image + const jpegBuffer = Buffer.from([ + 0xff, + 0xd8, + 0xff, + 0xe0, + 0x00, + 0x10, + 0x4a, + 0x46, // JPEG header + 0x49, + 0x46, + 0x00, + 0x01, + 0x01, + 0x00, + 0x00, + 0x01, + 0x00, + 0x01, + 0x00, + 0x00, + 0xff, + 0xd9, // End of image + ]); + const imagePath = path.join(testDir, "test.jpg"); + await fs.writeFile(imagePath, jpegBuffer); + + using testEnv = createTestFileReadTool({ cwd: testDir }); + const tool = testEnv.tool; + const args: FileReadToolArgs = { + filePath: imagePath, + }; + + // Execute + const result = (await tool.execute!(args, mockToolCallOptions)) as FileReadToolResult; + + // Assert execute result + expect(result.success).toBe(true); + if (result.success) { + expect(result.mime_type).toBe("image/jpeg"); + + // Test toModelOutput transformation + const modelOutput = tool.toModelOutput!(result); + expect(modelOutput.type).toBe("content"); + if (modelOutput.type === "content") { + expect(modelOutput.value).toHaveLength(1); + expect(modelOutput.value[0].type).toBe("media"); + if (modelOutput.value[0].type === "media") { + expect(modelOutput.value[0].mediaType).toBe("image/jpeg"); + expect(modelOutput.value[0].data).toBe(jpegBuffer.toString("base64")); + } + } + } + }); + + it("should return json for text files via toModelOutput", async () => { + // Setup + const content = "line one\nline two"; + await fs.writeFile(testFilePath, content); + + using testEnv = createTestFileReadTool({ cwd: testDir }); + const tool = testEnv.tool; + const args: FileReadToolArgs = { + filePath: testFilePath, + }; + + // Execute + const result = (await tool.execute!(args, mockToolCallOptions)) as FileReadToolResult; + + // Assert + expect(result.success).toBe(true); + if (result.success) { + // Test toModelOutput transformation + const modelOutput = tool.toModelOutput!(result); + expect(modelOutput.type).toBe("json"); + if (modelOutput.type === "json") { + expect(modelOutput.value).toEqual(result); + } + } + }); }); diff --git a/src/services/tools/file_read.ts b/src/services/tools/file_read.ts index 3c1227da6..4934df80a 100644 --- a/src/services/tools/file_read.ts +++ b/src/services/tools/file_read.ts @@ -1,6 +1,7 @@ import { tool } from "ai"; import * as fs from "fs/promises"; import * as path from "path"; +import * as mime from "mime-types"; import type { FileReadToolResult } from "@/types/tools"; import type { ToolConfiguration, ToolFactory } from "@/utils/tools/tools"; import { TOOL_DEFINITIONS } from "@/utils/tools/toolDefinitions"; @@ -15,6 +16,26 @@ export const createFileReadTool: ToolFactory = (config: ToolConfiguration) => { return tool({ description: TOOL_DEFINITIONS.file_read.description, inputSchema: TOOL_DEFINITIONS.file_read.schema, + toModelOutput: (output: FileReadToolResult) => { + // If this is an image file with a mime type, return it as media content + if (output.success && output.mime_type && output.mime_type.startsWith("image/")) { + return { + type: "content", + value: [ + { + type: "media", + data: output.content, + mediaType: output.mime_type, + }, + ], + }; + } + // Otherwise return as JSON (text files) + return { + type: "json", + value: output, + }; + }, execute: async ( { filePath, offset, limit }, { abortSignal: _abortSignal } @@ -53,7 +74,26 @@ export const createFileReadTool: ToolFactory = (config: ToolConfiguration) => { }; } - // Read full file content + // Detect MIME type + const mimeType = mime.lookup(resolvedPath) || undefined; + + // Check if this is a binary image file + if (mimeType && mimeType.startsWith("image/")) { + // Read as binary and encode as base64 for images + const buffer = await fs.readFile(resolvedPath); + const base64Content = buffer.toString("base64"); + + return { + success: true, + file_size: stats.size, + modifiedTime: stats.mtime.toISOString(), + lines_read: 0, // Images don't have lines + content: base64Content, + mime_type: mimeType, + }; + } + + // Read full file content as text for non-image files const fullContent = await fs.readFile(resolvedPath, { encoding: "utf-8" }); const startLineNumber = offset ?? 1; diff --git a/src/types/tools.ts b/src/types/tools.ts index d7be2038f..bcfe52d8d 100644 --- a/src/types/tools.ts +++ b/src/types/tools.ts @@ -50,6 +50,7 @@ export type FileReadToolResult = modifiedTime: string; lines_read: number; content: string; + mime_type?: string; } | { success: false; From 5dfc0e2cab5f7d89b0f1c8f2d34aaf1ef7279eb5 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 20 Oct 2025 18:51:26 -0400 Subject: [PATCH 2/4] Fix ESLint: use optional chaining for mime type checks --- src/services/tools/file_read.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/tools/file_read.ts b/src/services/tools/file_read.ts index 4934df80a..072eb582f 100644 --- a/src/services/tools/file_read.ts +++ b/src/services/tools/file_read.ts @@ -18,7 +18,7 @@ export const createFileReadTool: ToolFactory = (config: ToolConfiguration) => { inputSchema: TOOL_DEFINITIONS.file_read.schema, toModelOutput: (output: FileReadToolResult) => { // If this is an image file with a mime type, return it as media content - if (output.success && output.mime_type && output.mime_type.startsWith("image/")) { + if (output.success && output.mime_type?.startsWith("image/")) { return { type: "content", value: [ @@ -78,7 +78,7 @@ export const createFileReadTool: ToolFactory = (config: ToolConfiguration) => { const mimeType = mime.lookup(resolvedPath) || undefined; // Check if this is a binary image file - if (mimeType && mimeType.startsWith("image/")) { + if (mimeType?.startsWith("image/")) { // Read as binary and encode as base64 for images const buffer = await fs.readFile(resolvedPath); const base64Content = buffer.toString("base64"); From 0ca165b7cc4c8f6f43eb381fb1b0714f8da08307 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 20 Oct 2025 18:58:36 -0400 Subject: [PATCH 3/4] Pass tools to convertToModelMessages to enable toModelOutput This is the critical missing piece - without passing tools to convertToModelMessages, the toModelOutput function is never called and images are sent as JSON instead of media content. Added test to verify image tool results are converted to media content when tools are provided to convertToModelMessages. --- src/services/aiService.ts | 6 +- src/utils/messages/convertMessages.test.ts | 93 ++++++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 src/utils/messages/convertMessages.test.ts diff --git a/src/services/aiService.ts b/src/services/aiService.ts index 029f81a81..fcb812b21 100644 --- a/src/services/aiService.ts +++ b/src/services/aiService.ts @@ -455,9 +455,13 @@ export class AIService extends EventEmitter { log.debug_obj(`${workspaceId}/2a_redacted_messages.json`, redactedForProvider); // Convert CmuxMessage to ModelMessage format using Vercel AI SDK utility + // Pass earlyTools so convertToModelMessages can use toModelOutput for tool results + // (earlyTools has stub config but same tool definitions with toModelOutput functions) // Type assertion needed because CmuxMessage has custom tool parts for interrupted tools // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument - const modelMessages = convertToModelMessages(redactedForProvider as any); + const modelMessages = convertToModelMessages(redactedForProvider as any, { + tools: earlyTools, + }); log.debug_obj(`${workspaceId}/2_model_messages.json`, modelMessages); // Apply ModelMessage transforms based on provider requirements diff --git a/src/utils/messages/convertMessages.test.ts b/src/utils/messages/convertMessages.test.ts new file mode 100644 index 000000000..44ae91e23 --- /dev/null +++ b/src/utils/messages/convertMessages.test.ts @@ -0,0 +1,93 @@ +import { describe, it, expect } from "bun:test"; +import { convertToModelMessages } from "ai"; +import { createFileReadTool } from "@/services/tools/file_read"; +import type { UIMessage } from "ai"; +import * as fs from "fs"; +import * as path from "path"; +import * as os from "os"; + +describe("convertToModelMessages with tools", () => { + it("should use toModelOutput for image file_read results", async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "convert-test-")); + + try { + // Create a minimal PNG + const png = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, + 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, + 0x15, 0xc4, 0x89, 0x00, 0x00, 0x00, 0x0a, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9c, 0x63, 0x00, + 0x01, 0x00, 0x00, 0x05, 0x00, 0x01, 0x0d, 0x0a, 0x2d, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x49, + 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82, + ]); + const imgPath = path.join(tmpDir, "test.png"); + fs.writeFileSync(imgPath, png); + + // Create tool and execute + const tool = createFileReadTool({ cwd: tmpDir, tempDir: tmpDir }); + const result = await tool.execute!( + { filePath: imgPath }, + { toolCallId: "test", messages: [] } + ); + + // Create a message with tool result + const messages: UIMessage[] = [ + { + id: "1", + role: "user", + parts: [{ type: "text", text: "Read image" }], + }, + { + id: "2", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolCallId: "call_1", + toolName: "file_read", + state: "output-available", + input: { filePath: imgPath }, + output: result, + }, + ], + }, + ]; + + // Convert without tools - should get JSON + const withoutTools = convertToModelMessages(messages); + const toolMessage = withoutTools.find((m) => m.role === "tool"); + expect(toolMessage).toBeDefined(); + if (toolMessage && toolMessage.role === "tool") { + const content = toolMessage.content[0]; + expect(content.type).toBe("tool-result"); + if (content.type === "tool-result") { + // Without tools, output should be JSON + expect(content.output.type).toBe("json"); + } + } + + // Convert with tools - should use toModelOutput and get media content + const withTools = convertToModelMessages(messages, { + tools: { file_read: tool }, + }); + const toolMessageWithTools = withTools.find((m) => m.role === "tool"); + expect(toolMessageWithTools).toBeDefined(); + if (toolMessageWithTools && toolMessageWithTools.role === "tool") { + const content = toolMessageWithTools.content[0]; + expect(content.type).toBe("tool-result"); + if (content.type === "tool-result") { + // With tools, toModelOutput should convert images to media content + expect(content.output.type).toBe("content"); + if (content.output.type === "content") { + expect(content.output.value).toHaveLength(1); + expect(content.output.value[0].type).toBe("media"); + if (content.output.value[0].type === "media") { + expect(content.output.value[0].mediaType).toBe("image/png"); + } + } + } + } + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); From c72806715660a49bc3fe117074a26e3a04bcec39 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 20 Oct 2025 19:00:39 -0400 Subject: [PATCH 4/4] Fix ESLint errors in convertMessages test - Use async fs.promises.writeFile instead of sync - Use optional chaining for toolMessage checks - Add eslint-disable for unavoidable any assignment --- src/utils/messages/convertMessages.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/utils/messages/convertMessages.test.ts b/src/utils/messages/convertMessages.test.ts index 44ae91e23..0138b86d1 100644 --- a/src/utils/messages/convertMessages.test.ts +++ b/src/utils/messages/convertMessages.test.ts @@ -20,10 +20,11 @@ describe("convertToModelMessages with tools", () => { 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82, ]); const imgPath = path.join(tmpDir, "test.png"); - fs.writeFileSync(imgPath, png); + await fs.promises.writeFile(imgPath, png); // Create tool and execute const tool = createFileReadTool({ cwd: tmpDir, tempDir: tmpDir }); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const result = await tool.execute!( { filePath: imgPath }, { toolCallId: "test", messages: [] } @@ -56,7 +57,7 @@ describe("convertToModelMessages with tools", () => { const withoutTools = convertToModelMessages(messages); const toolMessage = withoutTools.find((m) => m.role === "tool"); expect(toolMessage).toBeDefined(); - if (toolMessage && toolMessage.role === "tool") { + if (toolMessage?.role === "tool") { const content = toolMessage.content[0]; expect(content.type).toBe("tool-result"); if (content.type === "tool-result") { @@ -71,7 +72,7 @@ describe("convertToModelMessages with tools", () => { }); const toolMessageWithTools = withTools.find((m) => m.role === "tool"); expect(toolMessageWithTools).toBeDefined(); - if (toolMessageWithTools && toolMessageWithTools.role === "tool") { + if (toolMessageWithTools?.role === "tool") { const content = toolMessageWithTools.content[0]; expect(content.type).toBe("tool-result"); if (content.type === "tool-result") {