feat(tools): add smart webfetch for agent-ready content extraction (#249)

* feat(tools): add modular smart webfetch

Port the previously personal and tested standalone smartfetch into slim as the built-in webfetch tool. This adds a modular implementation for fetch flow, extraction, cache handling, binary persistence, and secondary-model fallback, while wiring the tool into the plugin and adding the required runtime dependencies.

* test(tools): cover smart webfetch behavior

Add focused smartfetch tests for cache keys, redirect policy, llms.txt behavior, secondary-model fallback, and utility helpers. Also add a smartfetch codemap and document the webfetch tool in the global tools documentation.

* chore: keep repository checks green

Apply the small non-functional cleanup changes required to keep Biome and typechecking green alongside the smartfetch port: import ordering in config-io, removal of an unused import in the auto-update checker, and dead-field cleanup in the zellij multiplexer.

* fix(tools): address smart webfetch review findings

Update smartfetch to use an English-first Accept-Language default, simplify conditional revalidation headers, remove format from the fetch cache key, and make secondary-model resolution dynamic and portable. This keeps the fetch cache conservative on save-binary behavior while aligning config lookup more closely with the repo's runtime expectations.

---------

Co-authored-by: Alvin <alvin@cmngoal.com>

bun.lock

@@ -7,8 +7,12 @@
       "dependencies": {
         "@ast-grep/cli": "^0.40.0",
         "@modelcontextprotocol/sdk": "^1.26.0",
+        "@mozilla/readability": "^0.6.0",
         "@opencode-ai/plugin": "^1.2.6",
         "@opencode-ai/sdk": "^1.2.6",
+        "jsdom": "^26.1.0",
+        "lru-cache": "^11.2.2",
+        "turndown": "^7.2.0",
         "vscode-jsonrpc": "^8.2.0",
         "vscode-languageserver-protocol": "^3.17.5",
         "which": "^6.0.0",
@@ -16,6 +20,9 @@
       },
       "devDependencies": {
         "@biomejs/biome": "2.4.2",
+        "@types/jsdom": "^21.1.7",
+        "@types/node": "^24.6.1",
+        "@types/turndown": "^5.0.5",
         "@types/which": "^3.0.4",
         "all-contributors-cli": "^6.26.1",
         "bun-types": "1.3.9",
@@ -27,6 +34,8 @@
     "@ast-grep/cli",
   ],
   "packages": {
+    "@asamuzakjp/css-color": ["@asamuzakjp/css-color@3.2.0", "", { "dependencies": { "@csstools/css-calc": "^2.1.3", "@csstools/css-color-parser": "^3.0.9", "@csstools/css-parser-algorithms": "^3.0.4", "@csstools/css-tokenizer": "^3.0.3", "lru-cache": "^10.4.3" } }, "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw=="],
+
     "@ast-grep/cli": ["@ast-grep/cli@0.40.5", "", { "dependencies": { "detect-libc": "2.1.2" }, "optionalDependencies": { "@ast-grep/cli-darwin-arm64": "0.40.5", "@ast-grep/cli-darwin-x64": "0.40.5", "@ast-grep/cli-linux-arm64-gnu": "0.40.5", "@ast-grep/cli-linux-x64-gnu": "0.40.5", "@ast-grep/cli-win32-arm64-msvc": "0.40.5", "@ast-grep/cli-win32-ia32-msvc": "0.40.5", "@ast-grep/cli-win32-x64-msvc": "0.40.5" }, "bin": { "sg": "sg", "ast-grep": "ast-grep" } }, "sha512-yVXL7Gz0WIHerQLf+MVaVSkhIhidtWReG5akNVr/JS9OVCVkSdz7gWm7H8jVv2M9OO1tauuG76K3UaRGBPu5lQ=="],
 
     "@ast-grep/cli-darwin-arm64": ["@ast-grep/cli-darwin-arm64@0.40.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-T9CzwJ1GqQhnANdsu6c7iT1akpvTVMK+AZrxnhIPv33Ze5hrXUUkqan+j4wUAukRJDqU7u94EhXLSLD+5tcJ8g=="],
@@ -63,20 +72,42 @@
 
     "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.4.2", "", { "os": "win32", "cpu": "x64" }, "sha512-9ma7C4g8Sq3cBlRJD2yrsHXB1mnnEBdpy7PhvFrylQWQb4PoyCmPucdX7frvsSBQuFtIiKCrolPl/8tCZrKvgQ=="],
 
+    "@csstools/color-helpers": ["@csstools/color-helpers@5.1.0", "", {}, "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA=="],
+
+    "@csstools/css-calc": ["@csstools/css-calc@2.1.4", "", { "peerDependencies": { "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ=="],
+
+    "@csstools/css-color-parser": ["@csstools/css-color-parser@3.1.0", "", { "dependencies": { "@csstools/color-helpers": "^5.1.0", "@csstools/css-calc": "^2.1.4" }, "peerDependencies": { "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA=="],
+
+    "@csstools/css-parser-algorithms": ["@csstools/css-parser-algorithms@3.0.5", "", { "peerDependencies": { "@csstools/css-tokenizer": "^3.0.4" } }, "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ=="],
+
+    "@csstools/css-tokenizer": ["@csstools/css-tokenizer@3.0.4", "", {}, "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw=="],
+
     "@hono/node-server": ["@hono/node-server@1.19.9", "", { "peerDependencies": { "hono": "^4" } }, "sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw=="],
 
+    "@mixmark-io/domino": ["@mixmark-io/domino@2.2.0", "", {}, "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw=="],
+
     "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.26.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-Y5RmPncpiDtTXDbLKswIJzTqu2hyBKxTNsgKqKclDbhIgg1wgtf1fRuvxgTnRfcnxtvvgbIEcqUOzZrJ6iSReg=="],
 
+    "@mozilla/readability": ["@mozilla/readability@0.6.0", "", {}, "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ=="],
+
     "@opencode-ai/plugin": ["@opencode-ai/plugin@1.2.6", "", { "dependencies": { "@opencode-ai/sdk": "1.2.6", "zod": "4.1.8" } }, "sha512-CJEp3k17yWsjyfivm3zQof8L42pdze3a7iTqMOyesHgJplSuLiBYAMndbBYMDuJkyAh0dHYjw8v10vVw7Kfl4Q=="],
 
     "@opencode-ai/sdk": ["@opencode-ai/sdk@1.2.6", "", {}, "sha512-dWMF8Aku4h7fh8sw5tQ2FtbqRLbIFT8FcsukpxTird49ax7oUXP+gzqxM/VdxHjfksQvzLBjLZyMdDStc5g7xA=="],
 
-    "@types/node": ["@types/node@25.0.8", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-powIePYMmC3ibL0UJ2i2s0WIbq6cg6UyVFQxSCpaPxxzAaziRfimGivjdF943sSGV6RADVbk0Nvlm5P/FB44Zg=="],
+    "@types/jsdom": ["@types/jsdom@21.1.7", "", { "dependencies": { "@types/node": "*", "@types/tough-cookie": "*", "parse5": "^7.0.0" } }, "sha512-yOriVnggzrnQ3a9OKOCxaVuSug3w3/SbOj5i7VwXWZEyUNl3bLF9V3MfxGbZKuwqJOQyRfqXyROBB1CoZLFWzA=="],
+
+    "@types/node": ["@types/node@24.12.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g=="],
+
+    "@types/tough-cookie": ["@types/tough-cookie@4.0.5", "", {}, "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA=="],
+
+    "@types/turndown": ["@types/turndown@5.0.6", "", {}, "sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg=="],
 
     "@types/which": ["@types/which@3.0.4", "", {}, "sha512-liyfuo/106JdlgSchJzXEQCVArk0CvevqPote8F8HgWgJ3dRCcTHgJIsLDuee0kxk/mhbInzIZk3QWSZJ8R+2w=="],
 
     "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="],
 
+    "agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="],
+
     "ajv": ["ajv@8.17.1", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g=="],
 
     "ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="],
@@ -129,10 +160,16 @@
 
     "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
 
+    "cssstyle": ["cssstyle@4.6.0", "", { "dependencies": { "@asamuzakjp/css-color": "^3.2.0", "rrweb-cssom": "^0.8.0" } }, "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg=="],
+
+    "data-urls": ["data-urls@5.0.0", "", { "dependencies": { "whatwg-mimetype": "^4.0.0", "whatwg-url": "^14.0.0" } }, "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg=="],
+
     "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
 
     "decamelize": ["decamelize@1.2.0", "", {}, "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA=="],
 
+    "decimal.js": ["decimal.js@10.6.0", "", {}, "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg=="],
+
     "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="],
 
     "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="],
@@ -147,6 +184,8 @@
 
     "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="],
 
+    "entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
+
     "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="],
 
     "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="],
@@ -201,8 +240,14 @@
 
     "hono": ["hono@4.11.4", "", {}, "sha512-U7tt8JsyrxSRKspfhtLET79pU8K+tInj5QZXs1jSugO1Vq5dFj3kmZsRldo29mTBfcjDRVRXrEZ6LS63Cog9ZA=="],
 
+    "html-encoding-sniffer": ["html-encoding-sniffer@4.0.0", "", { "dependencies": { "whatwg-encoding": "^3.1.1" } }, "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ=="],
+
     "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="],
 
+    "http-proxy-agent": ["http-proxy-agent@7.0.2", "", { "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" } }, "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig=="],
+
+    "https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="],
+
     "iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="],
 
     "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
@@ -215,12 +260,16 @@
 
     "is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="],
 
+    "is-potential-custom-element-name": ["is-potential-custom-element-name@1.0.1", "", {}, "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ=="],
+
     "is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="],
 
     "isexe": ["isexe@4.0.0", "", {}, "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw=="],
 
     "jose": ["jose@6.1.3", "", {}, "sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ=="],
 
+    "jsdom": ["jsdom@26.1.0", "", { "dependencies": { "cssstyle": "^4.2.1", "data-urls": "^5.0.0", "decimal.js": "^10.5.0", "html-encoding-sniffer": "^4.0.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", "is-potential-custom-element-name": "^1.0.1", "nwsapi": "^2.2.16", "parse5": "^7.2.1", "rrweb-cssom": "^0.8.0", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^5.1.1", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^7.0.0", "whatwg-encoding": "^3.1.1", "whatwg-mimetype": "^4.0.0", "whatwg-url": "^14.1.1", "ws": "^8.18.0", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^3.0.0" }, "optionalPeers": ["canvas"] }, "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg=="],
+
     "json-fixer": ["json-fixer@1.6.15", "", { "dependencies": { "@babel/runtime": "^7.18.9", "chalk": "^4.1.2", "pegjs": "^0.10.0" } }, "sha512-TuDuZ5KrgyjoCIppdPXBMqiGfota55+odM+j2cQ5rt/XKyKmqGB3Whz1F8SN8+60yYGy/Nu5lbRZ+rx8kBIvBw=="],
 
     "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="],
@@ -231,6 +280,8 @@
 
     "lodash": ["lodash@4.17.23", "", {}, "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w=="],
 
+    "lru-cache": ["lru-cache@11.3.2", "", {}, "sha512-wgWa6FWQ3QRRJbIjbsldRJZxdxYngT/dO0I5Ynmlnin8qy7tC6xYzbcJjtN4wHLXtkbVwHzk0C+OejVw1XM+DQ=="],
+
     "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
 
     "media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="],
@@ -251,6 +302,8 @@
 
     "node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="],
 
+    "nwsapi": ["nwsapi@2.2.23", "", {}, "sha512-7wfH4sLbt4M0gCDzGE6vzQBo0bfTKjU7Sfpqy/7gs1qBfYz2vEJH6vXcBKpO3+6Yu1telwd0t9HpyOoLEQQbIQ=="],
+
     "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="],
 
     "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="],
@@ -269,6 +322,8 @@
 
     "p-try": ["p-try@2.2.0", "", {}, "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ=="],
 
+    "parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="],
+
     "parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="],
 
     "path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="],
@@ -287,6 +342,8 @@
 
     "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="],
 
+    "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="],
+
     "qs": ["qs@6.14.1", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ=="],
 
     "range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="],
@@ -303,12 +360,16 @@
 
     "router": ["router@2.2.0", "", { "dependencies": { "debug": "^4.4.0", "depd": "^2.0.0", "is-promise": "^4.0.0", "parseurl": "^1.3.3", "path-to-regexp": "^8.0.0" } }, "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ=="],
 
+    "rrweb-cssom": ["rrweb-cssom@0.8.0", "", {}, "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw=="],
+
     "run-async": ["run-async@2.4.1", "", {}, "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ=="],
 
     "rxjs": ["rxjs@6.6.7", "", { "dependencies": { "tslib": "^1.9.0" } }, "sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ=="],
 
     "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
 
+    "saxes": ["saxes@6.0.0", "", { "dependencies": { "xmlchars": "^2.2.0" } }, "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA=="],
+
     "send": ["send@1.2.1", "", { "dependencies": { "debug": "^4.4.3", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.1", "mime-types": "^3.0.2", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.2" } }, "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ=="],
 
     "serve-static": ["serve-static@2.2.1", "", { "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "parseurl": "^1.3.3", "send": "^1.2.0" } }, "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw=="],
@@ -339,16 +400,26 @@
 
     "supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="],
 
+    "symbol-tree": ["symbol-tree@3.2.4", "", {}, "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="],
+
     "through": ["through@2.3.8", "", {}, "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg=="],
 
+    "tldts": ["tldts@6.1.86", "", { "dependencies": { "tldts-core": "^6.1.86" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ=="],
+
+    "tldts-core": ["tldts-core@6.1.86", "", {}, "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA=="],
+
     "tmp": ["tmp@0.0.33", "", { "dependencies": { "os-tmpdir": "~1.0.2" } }, "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw=="],
 
     "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="],
 
-    "tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="],
+    "tough-cookie": ["tough-cookie@5.1.2", "", { "dependencies": { "tldts": "^6.1.32" } }, "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A=="],
+
+    "tr46": ["tr46@5.1.1", "", { "dependencies": { "punycode": "^2.3.1" } }, "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw=="],
 
     "tslib": ["tslib@1.14.1", "", {}, "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="],
 
+    "turndown": ["turndown@7.2.4", "", { "dependencies": { "@mixmark-io/domino": "^2.2.0" } }, "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ=="],
+
     "type-fest": ["type-fest@0.21.3", "", {}, "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w=="],
 
     "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="],
@@ -367,9 +438,15 @@
 
     "vscode-languageserver-types": ["vscode-languageserver-types@3.17.5", "", {}, "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg=="],
 
-    "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="],
+    "w3c-xmlserializer": ["w3c-xmlserializer@5.0.0", "", { "dependencies": { "xml-name-validator": "^5.0.0" } }, "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA=="],
+
+    "webidl-conversions": ["webidl-conversions@7.0.0", "", {}, "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="],
+
+    "whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="],
 
-    "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="],
+    "whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="],
+
+    "whatwg-url": ["whatwg-url@14.2.0", "", { "dependencies": { "tr46": "^5.1.0", "webidl-conversions": "^7.0.0" } }, "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw=="],
 
     "which": ["which@6.0.1", "", { "dependencies": { "isexe": "^4.0.0" }, "bin": { "node-which": "bin/which.js" } }, "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg=="],
 
@@ -379,6 +456,12 @@
 
     "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
 
+    "ws": ["ws@8.20.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA=="],
+
+    "xml-name-validator": ["xml-name-validator@5.0.0", "", {}, "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg=="],
+
+    "xmlchars": ["xmlchars@2.2.0", "", {}, "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="],
+
     "y18n": ["y18n@4.0.3", "", {}, "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ=="],
 
     "yargs": ["yargs@15.4.1", "", { "dependencies": { "cliui": "^6.0.0", "decamelize": "^1.2.0", "find-up": "^4.1.0", "get-caller-file": "^2.0.1", "require-directory": "^2.1.1", "require-main-filename": "^2.0.0", "set-blocking": "^2.0.0", "string-width": "^4.2.0", "which-module": "^2.0.0", "y18n": "^4.0.0", "yargs-parser": "^18.1.2" } }, "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A=="],
@@ -389,14 +472,28 @@
 
     "zod-to-json-schema": ["zod-to-json-schema@3.25.1", "", { "peerDependencies": { "zod": "^3.25 || ^4" } }, "sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA=="],
 
+    "@asamuzakjp/css-color/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="],
+
     "@opencode-ai/plugin/zod": ["zod@4.1.8", "", {}, "sha512-5R1P+WwQqmmMIEACyzSvo4JXHY5WiAFHRMg+zBZKgKS+Q1viRa0C1hmUKtHltoIFKtIdki3pRxkmpP74jnNYHQ=="],
 
+    "@types/jsdom/@types/node": ["@types/node@25.0.8", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-powIePYMmC3ibL0UJ2i2s0WIbq6cg6UyVFQxSCpaPxxzAaziRfimGivjdF943sSGV6RADVbk0Nvlm5P/FB44Zg=="],
+
+    "bun-types/@types/node": ["@types/node@25.0.8", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-powIePYMmC3ibL0UJ2i2s0WIbq6cg6UyVFQxSCpaPxxzAaziRfimGivjdF943sSGV6RADVbk0Nvlm5P/FB44Zg=="],
+
     "cross-spawn/which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
 
     "external-editor/iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="],
 
+    "node-fetch/whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="],
+
     "vscode-languageserver-protocol/vscode-jsonrpc": ["vscode-jsonrpc@8.2.0", "", {}, "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA=="],
 
+    "whatwg-encoding/iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="],
+
     "cross-spawn/which/isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="],
+
+    "node-fetch/whatwg-url/tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="],
+
+    "node-fetch/whatwg-url/webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="],
   }
 }

docs/tools.md

@@ -18,6 +18,18 @@ Background tasks integrate with [Multiplexer Integration](multiplexer-integratio
 
 ---
 
+## Web Fetch
+
+Fetch remote pages with content extraction tuned for docs/static sites.
+
+| Tool | Description |
+|------|-------------|
+| `webfetch` | Fetch a URL, optionally prefer `llms.txt`, extract main content from HTML, include metadata, and optionally save binary responses |
+
+`webfetch` blocks cross-origin redirects unless the requested URL or derived permission patterns explicitly allow them, and it can fall back to the raw fetched content when secondary-model summarization is unavailable.
+
+---
+
 ## LSP Tools
 
 Language Server Protocol integration for code intelligence across 30+ languages. OpenCode ships pre-configured LSP servers for TypeScript, Python, Rust, Go, and more.

package.json

@@ -54,10 +54,14 @@
     "release:major": "npm version major && git push --follow-tags && npm publish"
   },
   "dependencies": {
+    "@mozilla/readability": "^0.6.0",
     "@ast-grep/cli": "^0.40.0",
     "@modelcontextprotocol/sdk": "^1.26.0",
     "@opencode-ai/plugin": "^1.2.6",
     "@opencode-ai/sdk": "^1.2.6",
+    "jsdom": "^26.1.0",
+    "lru-cache": "^11.2.2",
+    "turndown": "^7.2.0",
     "vscode-jsonrpc": "^8.2.0",
     "vscode-languageserver-protocol": "^3.17.5",
     "which": "^6.0.0",
@@ -65,6 +69,9 @@
   },
   "devDependencies": {
     "@biomejs/biome": "2.4.2",
+    "@types/jsdom": "^21.1.7",
+    "@types/node": "^24.6.1",
+    "@types/turndown": "^5.0.5",
     "@types/which": "^3.0.4",
     "all-contributors-cli": "^6.26.1",
     "bun-types": "1.3.9",

src/cli/config-io.ts

@@ -6,13 +6,13 @@ import {
   statSync,
   writeFileSync,
 } from 'node:fs';
+import { pathToFileURL } from 'node:url';
 import {
   ensureConfigDir,
   ensureOpenCodeConfigDir,
   getExistingConfigPath,
   getLiteConfig,
 } from './paths';
-import { pathToFileURL } from 'node:url';
 import { generateLiteConfig } from './providers';
 import type {
   ConfigMergeResult,
@@ -42,7 +42,10 @@ function getPluginEntry(): string {
     const pluginEntryPath = cliEntryPath.match(
       /[\\/]dist[\\/]cli[\\/]index\.js$/,
     )
-      ? cliEntryPath.replace(/[\\/]dist[\\/]cli[\\/]index\.js$/, '/dist/index.js')
+      ? cliEntryPath.replace(
+          /[\\/]dist[\\/]cli[\\/]index\.js$/,
+          '/dist/index.js',
+        )
       : null;
 
     if (!pluginEntryPath) {

src/hooks/auto-update-checker/index.ts

@@ -7,7 +7,6 @@ import {
   getCachedVersion,
   getLatestVersion,
   getLocalDevVersion,
-  updatePinnedVersion,
 } from './checker';
 import { PACKAGE_NAME } from './constants';
 import type { AutoUpdateCheckerOptions } from './types';

src/index.ts

@@ -22,6 +22,7 @@ import {
   ast_grep_search,
   createBackgroundTools,
   createCouncilTool,
+  createWebfetchTool,
   lsp_diagnostics,
   lsp_find_references,
   lsp_goto_definition,
@@ -122,6 +123,7 @@ const OhMyOpenCodeLite: Plugin = async (ctx) => {
     : {};
 
   const mcps = createBuiltinMcps(config.disabled_mcps, config.websearch);
+  const webfetch = createWebfetchTool(ctx);
 
   // Initialize MultiplexerSessionManager to handle OpenCode's built-in Task tool sessions
   const multiplexerSessionManager = new MultiplexerSessionManager(
@@ -178,6 +180,7 @@ const OhMyOpenCodeLite: Plugin = async (ctx) => {
     tool: {
       ...backgroundTools,
       ...councilTools,
+      webfetch,
       ...todoContinuationHook.tool,
       lsp_goto_definition,
       lsp_find_references,

src/multiplexer/zellij/index.ts

@@ -23,8 +23,6 @@ export class ZellijMultiplexer implements Multiplexer {
 
   private binaryPath: string | null = null;
   private hasChecked = false;
-  private storedLayout: MultiplexerLayout;
-  private storedMainPaneSize: number;
   private agentTabId: string | null = null;
   private firstPaneId: string | null = null;
   private firstPaneUsed = false;
@@ -33,8 +31,8 @@ export class ZellijMultiplexer implements Multiplexer {
     // Note: Zellij does NOT support layout configuration like tmux.
     // These params are accepted for API consistency but are no-ops.
     // Zellij uses its own native layout algorithm for pane arrangement.
-    this.storedLayout = layout;
-    this.storedMainPaneSize = mainPaneSize;
+    void layout;
+    void mainPaneSize;
   }
 
   async isAvailable(): Promise<boolean> {

src/tools/index.ts

@@ -10,3 +10,4 @@ export {
   lspManager,
   setUserLspConfig,
 } from './lsp';
+export { createWebfetchTool } from './smartfetch';

src/tools/smartfetch/binary.ts

@@ -0,0 +1,58 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import type { BinaryFetch } from './types';
+
+function extensionForMime(contentType: string) {
+  const mime = contentType.split(';')[0]?.trim().toLowerCase();
+  const map: Record<string, string> = {
+    'application/pdf': 'pdf',
+    'image/png': 'png',
+    'image/jpeg': 'jpg',
+    'image/webp': 'webp',
+    'application/zip': 'zip',
+  };
+  return map[mime] || 'bin';
+}
+
+export function buildBinaryResultMessage(
+  fetchResult: BinaryFetch,
+  savedPath?: string,
+) {
+  const subject = fetchResult.binaryKind.toUpperCase();
+  if (savedPath) return `${subject} content saved to ${savedPath}`;
+  return `${subject} content omitted because it exceeds the download limit.`;
+}
+
+export async function saveBinary(
+  binaryDir: string,
+  data: Uint8Array,
+  contentType: string,
+  filename?: string,
+) {
+  await mkdir(binaryDir, { recursive: true });
+  const initialName =
+    filename || `webfetch-${Date.now()}.${extensionForMime(contentType)}`;
+  const parsed = path.parse(initialName);
+  for (let attempt = 0; attempt < 1000; attempt++) {
+    const candidateName =
+      attempt === 0
+        ? initialName
+        : `${parsed.name}-${attempt}${parsed.ext || `.${extensionForMime(contentType)}`}`;
+    const file = path.join(binaryDir, candidateName);
+    try {
+      await writeFile(file, data, { flag: 'wx' });
+      return file;
+    } catch (error: unknown) {
+      if (
+        typeof error === 'object' &&
+        error &&
+        'code' in error &&
+        error.code === 'EEXIST'
+      ) {
+        continue;
+      }
+      throw error;
+    }
+  }
+  throw new Error('Unable to allocate unique filename for binary content');
+}

src/tools/smartfetch/cache.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, test } from 'bun:test';
+import { buildCacheKey } from './cache';
+
+describe('smartfetch/cache', () => {
+  test('includes save_binary but not format in the cache key', () => {
+    const markdownKey = buildCacheKey(
+      'https://example.com/docs',
+      true,
+      'auto',
+      false,
+    );
+    const htmlKey = buildCacheKey(
+      'https://example.com/docs',
+      true,
+      'auto',
+      false,
+    );
+    const binaryKey = buildCacheKey(
+      'https://example.com/docs',
+      true,
+      'auto',
+      true,
+    );
+
+    expect(markdownKey).toBe(htmlKey);
+    expect(markdownKey).not.toBe(binaryKey);
+    expect(JSON.parse(markdownKey)).toMatchObject({
+      saveBinary: false,
+    });
+    expect(JSON.parse(binaryKey)).toMatchObject({
+      saveBinary: true,
+    });
+  });
+});

src/tools/smartfetch/cache.ts

@@ -0,0 +1,112 @@
+import { LRUCache } from 'lru-cache';
+import { canUseCanonicalCacheAlias, isHtmlLikeContentType } from './network';
+import type { FetchResult } from './types';
+
+export const CACHE = new LRUCache<string, FetchResult>({
+  maxSize: 50 * 1024 * 1024,
+  ttl: 15 * 60 * 1000,
+  sizeCalculation: (value) => {
+    if ('binary' in value) return value.data?.byteLength ?? 1024;
+    const rawContent =
+      value.rawContent ?? value.html ?? value.markdown ?? value.text ?? '';
+    return (
+      Buffer.byteLength(rawContent) +
+      Buffer.byteLength(value.html) +
+      Buffer.byteLength(value.markdown) +
+      Buffer.byteLength(value.text)
+    );
+  },
+});
+
+export function buildCacheKey(
+  url: string,
+  extractMain: boolean,
+  preferLlmsTxt: 'auto' | 'always' | 'never',
+  saveBinary: boolean,
+) {
+  const parsed = new URL(url);
+  return JSON.stringify({
+    url: parsed.toString(),
+    extractMain,
+    preferLlmsTxt,
+    saveBinary,
+  });
+}
+
+function cacheKeysFor(
+  fetchResult: FetchResult,
+  extractMain: boolean,
+  preferLlmsTxt: 'auto' | 'always' | 'never',
+  saveBinary: boolean,
+) {
+  const keys = new Set<string>();
+  keys.add(
+    buildCacheKey(
+      fetchResult.requestedUrl,
+      extractMain,
+      preferLlmsTxt,
+      saveBinary,
+    ),
+  );
+  keys.add(
+    buildCacheKey(fetchResult.finalUrl, extractMain, preferLlmsTxt, saveBinary),
+  );
+  if (
+    fetchResult.canonicalUrl &&
+    canUseCanonicalCacheAlias(fetchResult.finalUrl, fetchResult.canonicalUrl)
+  ) {
+    keys.add(
+      buildCacheKey(
+        fetchResult.canonicalUrl,
+        extractMain,
+        preferLlmsTxt,
+        saveBinary,
+      ),
+    );
+  }
+  return [...keys];
+}
+
+export function cacheFetchResult(
+  fetchResult: FetchResult,
+  extractMain: boolean,
+  preferLlmsTxt: 'auto' | 'always' | 'never',
+  saveBinary: boolean,
+) {
+  for (const key of cacheKeysFor(
+    fetchResult,
+    extractMain,
+    preferLlmsTxt,
+    saveBinary,
+  )) {
+    CACHE.set(key, fetchResult);
+  }
+}
+
+export function isInvalidLlmsResult(fetchResult: FetchResult | undefined) {
+  if (!fetchResult || 'binary' in fetchResult) return false;
+  if (!fetchResult.usedLlmsTxt || fetchResult.sourceKind !== 'llms_txt') {
+    return false;
+  }
+  const finalPath = (() => {
+    try {
+      return new URL(fetchResult.finalUrl).pathname.toLowerCase();
+    } catch {
+      return '';
+    }
+  })();
+  if (
+    !(finalPath.endsWith('/llms.txt') || finalPath.endsWith('/llms-full.txt'))
+  ) {
+    return true;
+  }
+  if (isHtmlLikeContentType(fetchResult.contentType)) return true;
+  if (/^\s*(<!doctype html|<html\b)/i.test(fetchResult.rawContent)) return true;
+  if (
+    /<title>\s*(log in|sign in|login)\b/i.test(fetchResult.rawContent) ||
+    /\blog[ -]?in\b/i.test(fetchResult.finalUrl)
+  ) {
+    return true;
+  }
+  return false;
+}

src/tools/smartfetch/codemap.md

@@ -0,0 +1,28 @@
+# src/tools/smartfetch/
+
+## Responsibility
+
+- Implement the built-in `webfetch` tool: fetch remote documents, enforce redirect/origin policy, probe `llms.txt` when useful, and return normalized text/markdown/html output (`tool.ts`, `network.ts`).
+- Handle content shaping around that fetch step: HTML extraction, metadata/frontmatter rendering, heading cleanup, cache keying, binary persistence, and secondary-model fallback (`utils.ts`, `cache.ts`, `binary.ts`, `secondary-model.ts`).
+
+## Design Patterns and Decisions
+
+- **One orchestration entrypoint:** `createWebfetchTool` in `tool.ts` owns permission prompts, cache lookup/revalidation, llms.txt preference logic, binary-vs-text branching, metadata emission, and optional secondary-model summarization.
+- **Transport/policy split from rendering:** `network.ts` focuses on URL normalization, redirect allowlists, charset/body decoding, header extraction, and llms.txt probing, while `utils.ts` focuses on turning fetched content into cleaned text/markdown/html plus frontmatter and user-facing messages.
+- **Cache keyed by fetch shape:** `cache.ts` keys fetches by URL plus behavior-affecting options (`extract_main`, `prefer_llms_txt`, `save_binary`), while render format is derived from the cached fetch result so text/markdown/html do not force redundant network requests.
+- **Graceful degradation:** missing/invalid `llms.txt`, blocked redirects, metadata-only binary responses, and secondary-model failures all return a usable result instead of throwing away the fetched content.
+
+## Data & Control Flow
+
+1. `createWebfetchTool` normalizes the requested URL, derives permission patterns/allowed origins, asks for `webfetch` permission, and computes the cache key (`tool.ts`, `network.ts`, `cache.ts`).
+2. If `prefer_llms_txt` applies, `probeLlmsText` tries `/llms-full.txt` then `/llms.txt`, following only permitted redirects and rejecting HTML/login-wall responses (`network.ts`).
+3. When the tool falls back to the page itself, `fetchWithUpgradeFallback` handles HTTPS upgrade fallback, redirect enforcement, conditional headers for revalidation, binary detection, and bounded body reads (`network.ts`, `tool.ts`).
+4. Text/HTML payloads are decoded and normalized through `extractFromHtml`, `cleanFetchedMarkdown`, `extractHeadingsFromMarkdown`, `frontmatter`, and `joinRenderedContent`; binary payloads optionally persist via `saveBinary` and return a metadata message (`utils.ts`, `binary.ts`, `tool.ts`).
+5. If the caller supplied a prompt and configured secondary models, `runSecondaryModelWithFallback` truncates input to a bounded size, disables tool access for the helper session, retries across configured models, and the tool degrades back to base fetched content if that step fails (`secondary-model.ts`, `tool.ts`).
+
+## Integration Points
+
+- `src/index.ts` registers the tool under the public name `webfetch`, so agents can call it alongside background, LSP, and AST-grep tools.
+- `src/tools/smartfetch/index.ts` re-exports the tool factory, description, and shared types for other modules or docs to import without reaching into implementation files.
+- `secondary-model.ts` depends on the OpenCode plugin client (`PluginInput['client']`) to spawn an isolated helper session, resolve `small_model` from the effective OpenCode config, and resolve `explorer` / `librarian` fallbacks from slim's own plugin config loader.
+- `cache.ts`, `network.ts`, and `utils.ts` are intentionally reusable seams for tests: cache behavior, redirect policy, llms probing, heading extraction, and render/metadata helpers can be verified without hitting the full tool entrypoint.

src/tools/smartfetch/constants.ts

@@ -0,0 +1,29 @@
+export const DOCS_HOST_SUFFIXES = [
+  '.readthedocs.io',
+  '.readthedocs.org',
+  '.gitbook.io',
+  '.netlify.app',
+  '.vercel.app',
+  'docs.rs',
+];
+
+export const DOCS_HOST_PREFIXES = ['docs.', 'developer.', 'dev.', 'wiki.'];
+export const MAX_REDIRECTS = 10;
+export const MAX_RESPONSE_BYTES = 10 * 1024 * 1024;
+export const MAX_BINARY_DOWNLOAD_BYTES = 2 * 1024 * 1024;
+export const DEFAULT_TIMEOUT_SECONDS = 30;
+export const MAX_TIMEOUT_SECONDS = 120;
+export const MAX_LLMS_PROBE_TIMEOUT_MS = 8000;
+export const MAX_MODEL_CONTENT_CHARS = 100_000;
+export const DEFAULT_ACCEPT_LANGUAGE = 'en;q=0.8,*;q=0.5';
+export const BINARY_PREFIXES = [
+  'image/',
+  'audio/',
+  'video/',
+  'application/pdf',
+  'application/zip',
+  'application/octet-stream',
+];
+
+export const WEBFETCH_DESCRIPTION =
+  'Fetch a URL with better extraction for static/docs pages. Supports llms.txt probing, content-focused HTML extraction, metadata, redirects, and an optional prompt processed by a cheap secondary model.';

src/tools/smartfetch/index.ts

@@ -0,0 +1,8 @@
+export { WEBFETCH_DESCRIPTION } from './constants';
+export { createWebfetchTool } from './tool';
+export type {
+  BinaryFetch,
+  CachedFetch,
+  FetchResult,
+  SmartfetchOptions,
+} from './types';

src/tools/smartfetch/network.test.ts

@@ -0,0 +1,178 @@
+import { afterEach, describe, expect, mock, test } from 'bun:test';
+import {
+  buildAllowedOrigins,
+  buildConditionalHeaders,
+  fetchWithRedirects,
+} from './network';
+
+describe('smartfetch/network', () => {
+  const originalFetch = globalThis.fetch;
+
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+    mock.restore();
+  });
+
+  test('collects unique allowed origins from permission patterns', () => {
+    const origins = [
+      ...buildAllowedOrigins([
+        'https://docs.example.com/page',
+        'https://docs.example.com/llms.txt',
+        'https://cdn.example.com/asset',
+        'not-a-url',
+      ]),
+    ].sort();
+
+    expect(origins).toEqual([
+      'https://cdn.example.com',
+      'https://docs.example.com',
+    ]);
+  });
+
+  test('follows permitted same-origin redirects', async () => {
+    const fetchMock = mock(async (input: string | URL | Request) => {
+      const url = typeof input === 'string' ? input : input.toString();
+
+      if (url === 'https://docs.example.com/start') {
+        return new Response('', {
+          status: 302,
+          headers: { location: '/next' },
+        });
+      }
+
+      if (url === 'https://docs.example.com/next') {
+        return new Response('ok', {
+          status: 200,
+          headers: { 'content-type': 'text/plain' },
+        });
+      }
+
+      throw new Error(`Unexpected fetch URL: ${url}`);
+    });
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    const result = await fetchWithRedirects(
+      'https://docs.example.com/start',
+      1_000,
+      'markdown',
+      new AbortController().signal,
+    );
+
+    expect('blockedRedirect' in result).toBe(false);
+    if ('blockedRedirect' in result) {
+      throw new Error('Expected redirect to be followed');
+    }
+
+    expect(result.finalUrl).toBe('https://docs.example.com/next');
+    expect(result.redirectChain).toEqual([
+      {
+        from: 'https://docs.example.com/start',
+        to: 'https://docs.example.com/next',
+        status: 302,
+      },
+    ]);
+  });
+
+  test('blocks cross-origin redirects when the origin is not allowed', async () => {
+    const fetchMock = mock(async (input: string | URL | Request) => {
+      const url = typeof input === 'string' ? input : input.toString();
+
+      if (url === 'https://docs.example.com/start') {
+        return new Response('', {
+          status: 302,
+          headers: { location: 'https://other.example.com/landing' },
+        });
+      }
+
+      throw new Error(`Unexpected fetch URL: ${url}`);
+    });
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    const result = await fetchWithRedirects(
+      'https://docs.example.com/start',
+      1_000,
+      'markdown',
+      new AbortController().signal,
+    );
+
+    expect(result).toEqual({
+      blockedRedirect: true,
+      redirectUrl: 'https://other.example.com/landing',
+      statusCode: 302,
+      redirectChain: [
+        {
+          from: 'https://docs.example.com/start',
+          to: 'https://other.example.com/landing',
+          status: 302,
+        },
+      ],
+    });
+  });
+
+  test('allows redirects to explicitly allowed origins', async () => {
+    const fetchMock = mock(async (input: string | URL | Request) => {
+      const url = typeof input === 'string' ? input : input.toString();
+
+      if (url === 'https://docs.example.com/start') {
+        return new Response('', {
+          status: 302,
+          headers: { location: 'https://cdn.example.com/asset' },
+        });
+      }
+
+      if (url === 'https://cdn.example.com/asset') {
+        return new Response('ok', {
+          status: 200,
+          headers: { 'content-type': 'text/plain' },
+        });
+      }
+
+      throw new Error(`Unexpected fetch URL: ${url}`);
+    });
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    const result = await fetchWithRedirects(
+      'https://docs.example.com/start',
+      1_000,
+      'markdown',
+      new AbortController().signal,
+      undefined,
+      'GET',
+      new Set(['https://docs.example.com', 'https://cdn.example.com']),
+    );
+
+    expect('blockedRedirect' in result).toBe(false);
+    if ('blockedRedirect' in result) {
+      throw new Error('Expected redirect to be allowed');
+    }
+
+    expect(result.finalUrl).toBe('https://cdn.example.com/asset');
+  });
+
+  test('builds conditional headers from etag and last-modified without binary branching', () => {
+    expect(buildConditionalHeaders(undefined)).toBeUndefined();
+
+    expect(
+      buildConditionalHeaders({
+        requestedUrl: 'https://example.com/file',
+        finalUrl: 'https://example.com/file',
+        statusCode: 200,
+        contentType: 'application/pdf',
+        charset: undefined,
+        etag: '"abc"',
+        lastModified: 'Wed, 01 Jan 2025 00:00:00 GMT',
+        contentLength: 42,
+        filename: 'file.pdf',
+        canonicalUrl: 'https://example.com/file',
+        redirectChain: [],
+        upgradedToHttps: false,
+        truncated: false,
+        binary: true,
+        binaryKind: 'pdf',
+      }),
+    ).toEqual({
+      'If-None-Match': '"abc"',
+      'If-Modified-Since': 'Wed, 01 Jan 2025 00:00:00 GMT',
+    });
+  });
+});

src/tools/smartfetch/network.ts

@@ -0,0 +1,614 @@
+import path from 'node:path';
+import {
+  BINARY_PREFIXES,
+  DEFAULT_ACCEPT_LANGUAGE,
+  DOCS_HOST_PREFIXES,
+  DOCS_HOST_SUFFIXES,
+  MAX_REDIRECTS,
+  MAX_RESPONSE_BYTES,
+} from './constants';
+import type {
+  BinaryFetch,
+  DecodedBody,
+  FetchResult,
+  FetchWithRedirectsResult,
+  LlmsProbeResult,
+} from './types';
+import { trimBlankRuns } from './utils';
+
+export function normalizeUrl(input: string) {
+  const parsed = new URL(input);
+  const originalUrl = parsed.toString();
+  let upgradedToHttps = false;
+  let fallbackUrl: string | undefined;
+  if (parsed.protocol === 'http:') {
+    fallbackUrl = originalUrl;
+    parsed.protocol = 'https:';
+    upgradedToHttps = true;
+  }
+  return { url: parsed.toString(), upgradedToHttps, fallbackUrl, originalUrl };
+}
+
+export function isDocsLikeUrl(url: URL) {
+  const host = url.hostname.toLowerCase();
+  return (
+    DOCS_HOST_SUFFIXES.some((suffix) => host.endsWith(suffix)) ||
+    DOCS_HOST_PREFIXES.some((prefix) => host.startsWith(prefix))
+  );
+}
+
+export function buildPermissionPatterns(
+  normalized: ReturnType<typeof normalizeUrl>,
+  shouldProbeLlmsTxt: boolean,
+) {
+  const patterns = new Set<string>([normalized.url]);
+  const origins = [new URL(normalized.url).origin];
+  if (normalized.fallbackUrl) {
+    patterns.add(normalized.fallbackUrl);
+    origins.push(new URL(normalized.fallbackUrl).origin);
+  }
+  if (shouldProbeLlmsTxt) {
+    for (const origin of origins) {
+      patterns.add(`${origin}/llms-full.txt`);
+      patterns.add(`${origin}/llms.txt`);
+    }
+  }
+  return [...patterns];
+}
+
+export function buildAllowedOrigins(patterns: string[]) {
+  const origins = new Set<string>();
+  for (const pattern of patterns) {
+    try {
+      origins.add(new URL(pattern).origin);
+    } catch {
+      // ignore invalid patterns
+    }
+  }
+  return origins;
+}
+
+export function canUseCanonicalCacheAlias(baseUrl: string, aliasUrl: string) {
+  try {
+    const base = new URL(baseUrl);
+    const alias = new URL(aliasUrl);
+    if (alias.username || alias.password) return false;
+    return (
+      base.protocol === alias.protocol &&
+      base.hostname === alias.hostname &&
+      base.port === alias.port &&
+      base.pathname === alias.pathname &&
+      base.search === alias.search
+    );
+  } catch {
+    return false;
+  }
+}
+
+function isPermittedRedirect(
+  from: string,
+  to: string,
+  allowedOrigins?: Set<string>,
+) {
+  try {
+    const a = new URL(from);
+    const b = new URL(to);
+    if (a.protocol !== b.protocol) return false;
+    if (a.port !== b.port) return false;
+    if (b.username || b.password) return false;
+    if (allowedOrigins) return allowedOrigins.has(b.origin);
+    return a.origin === b.origin;
+  } catch {
+    return false;
+  }
+}
+
+export function isBinaryContentType(contentType: string) {
+  const mime = contentType.split(';')[0]?.trim().toLowerCase() || '';
+  return BINARY_PREFIXES.some((prefix) => mime.startsWith(prefix));
+}
+
+export function getBinaryKind(contentType: string): BinaryFetch['binaryKind'] {
+  const mime = contentType.split(';')[0]?.trim().toLowerCase() || '';
+  if (mime.startsWith('image/')) return 'image';
+  if (mime.startsWith('audio/')) return 'audio';
+  if (mime.startsWith('video/')) return 'video';
+  if (mime === 'application/pdf') return 'pdf';
+  return 'binary';
+}
+
+function acceptHeader(_format: 'text' | 'markdown' | 'html') {
+  return 'text/html;q=1.0, application/xhtml+xml;q=0.9, text/markdown;q=0.8, text/plain;q=0.8, */*;q=0.1';
+}
+
+function inferCharsetFromHtml(text: string) {
+  const metaCharset = text.match(
+    /<meta[^>]+charset\s*=\s*["']?([^\s"'>/;]+)/i,
+  )?.[1];
+  if (metaCharset) return metaCharset.trim();
+  const httpEquiv = text.match(
+    /<meta[^>]+http-equiv\s*=\s*["']content-type["'][^>]+content\s*=\s*["'][^"']*charset=([^\s"'>;]+)/i,
+  )?.[1];
+  if (httpEquiv) return httpEquiv.trim();
+  return undefined;
+}
+
+export function looksLikeHtmlText(text: string) {
+  return /^\s*(<!doctype html|<html\b|<head\b|<body\b)/i.test(text);
+}
+
+function isLikelyDecodedText(text: string) {
+  if (!text) return false;
+  let suspicious = 0;
+  let printable = 0;
+  for (const char of text.slice(0, 2048)) {
+    const code = char.charCodeAt(0);
+    const isWhitespace =
+      code === 9 || code === 10 || code === 13 || code === 32;
+    const isControl = code < 32 && !isWhitespace;
+    if (isControl) suspicious++;
+    else printable++;
+  }
+  const total = Math.max(printable + suspicious, 1);
+  return suspicious / total < 0.02 && printable / total > 0.85;
+}
+
+function tryDecodeWithCharset(data: Uint8Array, charset: string) {
+  try {
+    return new TextDecoder(
+      charset,
+      charset.toLowerCase() === 'utf-8' ? { fatal: true } : undefined,
+    ).decode(data);
+  } catch {
+    return undefined;
+  }
+}
+
+function detectBestEffortCharset(data: Uint8Array) {
+  for (const charset of ['utf-8', 'windows-1252', 'iso-8859-1']) {
+    const decoded = tryDecodeWithCharset(data, charset);
+    if (decoded && isLikelyDecodedText(decoded)) {
+      return { charset, text: decoded };
+    }
+  }
+  return undefined;
+}
+
+export async function runWithScopedTimeout<T>(
+  parentSignal: AbortSignal,
+  timeoutMs: number,
+  fn: (signal: AbortSignal) => Promise<T>,
+) {
+  const controller = new AbortController();
+  const abortHandler = () => controller.abort(parentSignal.reason);
+  if (parentSignal.aborted) controller.abort(parentSignal.reason);
+  else parentSignal.addEventListener('abort', abortHandler, { once: true });
+  const timeout = setTimeout(
+    () => controller.abort(new Error(`timeout after ${timeoutMs}ms`)),
+    timeoutMs,
+  );
+  try {
+    return await fn(controller.signal);
+  } finally {
+    clearTimeout(timeout);
+    parentSignal.removeEventListener('abort', abortHandler);
+  }
+}
+
+export async function readBodyLimited(
+  response: Response,
+  maxBytes = MAX_RESPONSE_BYTES,
+) {
+  if (!response.body) return { data: new Uint8Array(), truncated: false };
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  let truncated = false;
+
+  while (true) {
+    const { value, done } = await reader.read();
+    if (done) break;
+    if (!value) continue;
+    if (total + value.byteLength > maxBytes) {
+      const allowed = maxBytes - total;
+      if (allowed > 0) chunks.push(value.slice(0, allowed));
+      truncated = true;
+      try {
+        await reader.cancel();
+      } catch {
+        // ignore cancel failures
+      }
+      break;
+    }
+    chunks.push(value);
+    total += value.byteLength;
+  }
+
+  const merged = new Uint8Array(
+    chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0),
+  );
+  let offset = 0;
+  for (const chunk of chunks) {
+    merged.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return { data: merged, truncated };
+}
+
+export async function fetchWithRedirects(
+  url: string,
+  _timeoutMs: number,
+  format: 'text' | 'markdown' | 'html',
+  signal: AbortSignal,
+  extraHeaders?: Record<string, string>,
+  method: 'GET' | 'HEAD' = 'GET',
+  allowedOrigins?: Set<string>,
+): Promise<FetchWithRedirectsResult> {
+  const redirects = [];
+  let current = url;
+
+  for (let i = 0; i < MAX_REDIRECTS; i++) {
+    const response = await fetch(current, {
+      redirect: 'manual',
+      signal,
+      method,
+      headers: {
+        'User-Agent': 'opencode-smartfetch/1.0',
+        Accept: acceptHeader(format),
+        'Accept-Language': DEFAULT_ACCEPT_LANGUAGE,
+        ...extraHeaders,
+      },
+    });
+
+    if (response.status >= 300 && response.status < 400) {
+      const location = response.headers.get('location');
+      if (!location) {
+        throw new Error(
+          `Redirect response missing location header: ${response.status}`,
+        );
+      }
+      const next = new URL(location, current).toString();
+      redirects.push({ from: current, to: next, status: response.status });
+      if (!isPermittedRedirect(current, next, allowedOrigins)) {
+        try {
+          await response.body?.cancel();
+        } catch {
+          // ignore cancel failures
+        }
+        return {
+          blockedRedirect: true,
+          redirectUrl: next,
+          statusCode: response.status,
+          redirectChain: redirects,
+        };
+      }
+      try {
+        await response.body?.cancel();
+      } catch {
+        // ignore cancel failures
+      }
+      current = next;
+      continue;
+    }
+
+    return { response, finalUrl: current, redirectChain: redirects };
+  }
+
+  throw new Error(`Too many redirects (exceeded ${MAX_REDIRECTS})`);
+}
+
+export async function fetchWithUpgradeFallback(
+  normalized: ReturnType<typeof normalizeUrl>,
+  timeoutMs: number,
+  format: 'text' | 'markdown' | 'html',
+  signal: AbortSignal,
+  extraHeaders?: Record<string, string>,
+  method: 'GET' | 'HEAD' = 'GET',
+  allowedOrigins?: Set<string>,
+) {
+  try {
+    const result = await fetchWithRedirects(
+      normalized.url,
+      timeoutMs,
+      format,
+      signal,
+      extraHeaders,
+      method,
+      allowedOrigins,
+    );
+    if (normalized.fallbackUrl && 'blockedRedirect' in result) {
+      const fallbackResult = await fetchWithRedirects(
+        normalized.fallbackUrl,
+        timeoutMs,
+        format,
+        signal,
+        extraHeaders,
+        method,
+        allowedOrigins,
+      );
+      return { result: fallbackResult, upgradedToHttps: false };
+    }
+    if (
+      normalized.fallbackUrl &&
+      !('blockedRedirect' in result) &&
+      result.response.status !== 304 &&
+      !result.response.ok
+    ) {
+      const fallbackResult = await fetchWithRedirects(
+        normalized.fallbackUrl,
+        timeoutMs,
+        format,
+        signal,
+        extraHeaders,
+        method,
+        allowedOrigins,
+      );
+      return { result: fallbackResult, upgradedToHttps: false };
+    }
+    return { result, upgradedToHttps: normalized.upgradedToHttps };
+  } catch (error) {
+    if (!normalized.fallbackUrl) throw error;
+    const result = await fetchWithRedirects(
+      normalized.fallbackUrl,
+      timeoutMs,
+      format,
+      signal,
+      extraHeaders,
+      method,
+      allowedOrigins,
+    );
+    return { result, upgradedToHttps: false };
+  }
+}
+
+function parseContentLength(headers: Headers) {
+  const raw = headers.get('content-length');
+  if (!raw) return undefined;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined;
+}
+
+function parseCharset(contentType: string) {
+  const match = contentType.match(/charset\s*=\s*([^;]+)/i);
+  return match?.[1]?.trim().replace(/^['"]|['"]$/g, '') || undefined;
+}
+
+export function isHtmlLikeContentType(contentType: string) {
+  const mime = contentType.split(';')[0]?.trim().toLowerCase() || '';
+  return mime === 'text/html' || mime === 'application/xhtml+xml';
+}
+
+export function decodeBody(
+  data: Uint8Array,
+  charset: string | undefined,
+  contentType?: string,
+): DecodedBody {
+  let declaredCharset = charset?.trim() || undefined;
+  const utf8Text = new TextDecoder().decode(data);
+
+  if (!declaredCharset && contentType && isHtmlLikeContentType(contentType)) {
+    declaredCharset = inferCharsetFromHtml(utf8Text);
+  }
+
+  if (!declaredCharset) {
+    const detected = detectBestEffortCharset(data);
+    if (detected && detected.charset !== 'utf-8') {
+      return {
+        text: detected.text,
+        decodedCharset: detected.charset,
+        decodeFallback: true,
+        decodeWarning: `Guessed charset without declaration: ${detected.charset}`,
+      };
+    }
+    return {
+      text: utf8Text,
+      decodedCharset: 'utf-8',
+      decodeFallback: false,
+      decodeWarning: undefined,
+    };
+  }
+
+  try {
+    return {
+      text: new TextDecoder(declaredCharset).decode(data),
+      decodedCharset: declaredCharset,
+      decodeFallback: false,
+      decodeWarning: undefined,
+    };
+  } catch {
+    return {
+      text: utf8Text,
+      decodedCharset: 'utf-8',
+      decodeFallback: true,
+      decodeWarning: `Unsupported charset decoder: ${declaredCharset}`,
+    };
+  }
+}
+
+export function looksLikeTextBody(data: Uint8Array) {
+  if (!data.byteLength) return true;
+  const sample = data.slice(0, Math.min(data.byteLength, 2048));
+  if (detectBestEffortCharset(sample)) return true;
+
+  let suspicious = 0;
+  let printableAscii = 0;
+  for (const byte of sample) {
+    if (byte === 0) return false;
+    const isWhitespace = byte === 9 || byte === 10 || byte === 13;
+    const isPrintableAscii = byte >= 32 && byte <= 126;
+    if (isWhitespace || isPrintableAscii) printableAscii++;
+    if (!isWhitespace && !isPrintableAscii) suspicious++;
+  }
+  return (
+    suspicious / sample.byteLength < 0.02 &&
+    printableAscii / sample.byteLength > 0.85
+  );
+}
+
+export function isGenericBinaryMime(contentType: string) {
+  const mime = contentType.split(';')[0]?.trim().toLowerCase() || '';
+  return mime === 'application/octet-stream';
+}
+
+function parseFilenameFromContentDisposition(value: string | null) {
+  if (!value) return undefined;
+  const utf8 = value.match(/filename\*\s*=\s*UTF-8''([^;]+)/i);
+  if (utf8?.[1]) {
+    try {
+      return decodeURIComponent(utf8[1].trim().replace(/^"|"$/g, ''));
+    } catch {
+      // ignore invalid encoding
+    }
+  }
+  const basic = value.match(/filename\s*=\s*("?)([^";]+)\1/i);
+  if (basic?.[2]) return basic[2].trim();
+  return undefined;
+}
+
+function inferFilenameFromUrl(url: string) {
+  try {
+    const parsed = new URL(url);
+    const last = parsed.pathname.split('/').filter(Boolean).pop();
+    if (!last || !last.includes('.')) return undefined;
+    return decodeURIComponent(last);
+  } catch {
+    return undefined;
+  }
+}
+
+function truncateFilename(name: string, maxLength = 180) {
+  if (name.length <= maxLength) return name;
+  const parsed = path.parse(name);
+  const ext = parsed.ext || '';
+  const baseLimit = Math.max(1, maxLength - ext.length);
+  return `${parsed.name.slice(0, baseLimit)}${ext}`;
+}
+
+function sanitizeFilename(name: string) {
+  let sanitized = Array.from(name, (char) => {
+    const code = char.charCodeAt(0);
+    if (code < 32 || '<>:"/\\|?*'.includes(char)) return '_';
+    return char;
+  }).join('');
+  sanitized = sanitized.replace(/\s+/g, ' ').trim();
+  sanitized = sanitized.replace(/^[.\s]+|[.\s]+$/g, '');
+  if (!sanitized) sanitized = 'download';
+  if (/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?$/i.test(sanitized)) {
+    sanitized = `_${sanitized}`;
+  }
+  return truncateFilename(sanitized);
+}
+
+export function extractHeaderMetadata(headers: Headers, finalUrl: string) {
+  const filename =
+    parseFilenameFromContentDisposition(headers.get('content-disposition')) ||
+    inferFilenameFromUrl(finalUrl);
+  const contentType = headers.get('content-type') || '';
+  return {
+    contentType: contentType || undefined,
+    charset: parseCharset(contentType),
+    etag: headers.get('etag') || undefined,
+    lastModified: headers.get('last-modified') || undefined,
+    contentLength: parseContentLength(headers),
+    filename: filename ? sanitizeFilename(filename) : undefined,
+  };
+}
+
+export function buildConditionalHeaders(cached: FetchResult | undefined) {
+  if (!cached || (!cached.etag && !cached.lastModified)) {
+    return undefined;
+  }
+  const headers: Record<string, string> = {};
+  if (cached.etag) headers['If-None-Match'] = cached.etag;
+  if (cached.lastModified) headers['If-Modified-Since'] = cached.lastModified;
+  return Object.keys(headers).length ? headers : undefined;
+}
+
+export async function probeLlmsText(
+  url: URL,
+  timeoutMs: number,
+  signal: AbortSignal,
+  fallbackOrigin?: string,
+): Promise<LlmsProbeResult> {
+  const origins = [`${url.protocol}//${url.host}`];
+  if (fallbackOrigin && !origins.includes(fallbackOrigin)) {
+    origins.push(fallbackOrigin);
+  }
+  const allowedOrigins = new Set(origins);
+  let lastError: string | undefined;
+  for (const candidate of origins.flatMap((origin) => [
+    `${origin}/llms-full.txt`,
+    `${origin}/llms.txt`,
+  ])) {
+    try {
+      const result = await fetchWithRedirects(
+        candidate,
+        timeoutMs,
+        'markdown',
+        signal,
+        {
+          Accept: 'text/plain, text/markdown;q=0.9, */*;q=0.1',
+        },
+        'GET',
+        allowedOrigins,
+      );
+      if ('blockedRedirect' in result) {
+        lastError = `llms.txt probe blocked by cross-host redirect: ${result.redirectUrl}`;
+        continue;
+      }
+      const { response, finalUrl, redirectChain } = result;
+      if (!response.ok) {
+        try {
+          await response.body?.cancel();
+        } catch {
+          // ignore cancel failures
+        }
+        continue;
+      }
+      const headers = extractHeaderMetadata(response.headers, finalUrl);
+      const body = await readBodyLimited(response, MAX_RESPONSE_BYTES);
+      const decoded = decodeBody(
+        body.data,
+        headers.charset,
+        headers.contentType,
+      );
+      const text = decoded.text;
+      const finalPath = new URL(finalUrl).pathname.toLowerCase();
+      const contentType = (headers.contentType || '').toLowerCase();
+      const looksLikeLlmsPath =
+        finalPath.endsWith('/llms.txt') || finalPath.endsWith('/llms-full.txt');
+      const looksHtml =
+        contentType.includes('text/html') ||
+        contentType.includes('application/xhtml+xml');
+      const looksLikeHtmlBody = /^\s*(<!doctype html|<html\b)/i.test(text);
+      const looksLikeLoginWall =
+        /<title>\s*(log in|sign in|login)\b/i.test(text) ||
+        /\blog[ -]?in\b/i.test(finalUrl);
+      if (!looksLikeLlmsPath) {
+        lastError = `llms.txt probe resolved to non-llms path: ${finalUrl}`;
+        continue;
+      }
+      if (looksHtml || looksLikeHtmlBody || looksLikeLoginWall) {
+        lastError = `llms.txt probe returned HTML/login content: ${finalUrl}`;
+        continue;
+      }
+      if (text.trim()) {
+        return {
+          url: finalUrl,
+          statusCode: response.status,
+          redirectChain,
+          text: trimBlankRuns(text),
+          headers,
+          truncated: body.truncated,
+          decodedCharset: decoded.decodedCharset,
+          decodeFallback: decoded.decodeFallback,
+          decodeWarning: decoded.decodeWarning,
+          upgradedToHttps: candidate.startsWith('https://') && !!fallbackOrigin,
+        };
+      }
+    } catch (error: unknown) {
+      lastError = error instanceof Error ? error.message : String(error);
+    }
+  }
+  return { error: lastError };
+}

src/tools/smartfetch/secondary-model.test.ts

@@ -0,0 +1,85 @@
+import { afterEach, describe, expect, mock, test } from 'bun:test';
+import { runSecondaryModelWithFallback } from './secondary-model';
+import type { SecondaryModel } from './types';
+
+type PromptStep = {
+  text?: string;
+  error?: Error;
+};
+
+function createMockClient(steps: PromptStep[]) {
+  let createCount = 0;
+  let promptCount = 0;
+
+  return {
+    session: {
+      create: mock(async () => ({ id: `session-${createCount++}` })),
+      prompt: mock(async () => {
+        const step = steps[promptCount++] ?? {};
+        if (step.error) {
+          throw step.error;
+        }
+        return {
+          data: {
+            parts: [{ type: 'text', text: step.text ?? '' }],
+          },
+        };
+      }),
+      delete: mock(async () => ({})),
+    },
+    tool: {
+      ids: mock(async () => ({ data: ['read', 'bash'] })),
+    },
+  } as any;
+}
+
+describe('smartfetch/secondary-model', () => {
+  const models: SecondaryModel[] = [
+    { providerID: 'provider-a', modelID: 'small' },
+    { providerID: 'provider-b', modelID: 'fallback' },
+  ];
+
+  afterEach(() => {
+    mock.restore();
+  });
+
+  test('falls back when the first model returns empty text', async () => {
+    const client = createMockClient([
+      { text: '   ' },
+      { text: 'Useful answer' },
+    ]);
+
+    const result = await runSecondaryModelWithFallback(
+      client,
+      '/tmp/project',
+      models,
+      'Summarize the page',
+      'This is enough fetched content to clear the short-content guard.',
+    );
+
+    expect(result.text).toBe('Useful answer');
+    expect(result.model).toEqual(models[1]);
+    expect(client.session.prompt).toHaveBeenCalledTimes(2);
+    expect(client.session.delete).toHaveBeenCalledTimes(2);
+  });
+
+  test('falls back when the first model throws', async () => {
+    const client = createMockClient([
+      { error: new Error('primary failed') },
+      { text: 'Recovered answer' },
+    ]);
+
+    const result = await runSecondaryModelWithFallback(
+      client,
+      '/tmp/project',
+      models,
+      'Extract the answer',
+      'This is enough fetched content to clear the short-content guard.',
+    );
+
+    expect(result.text).toBe('Recovered answer');
+    expect(result.model).toEqual(models[1]);
+    expect(client.session.prompt).toHaveBeenCalledTimes(2);
+    expect(client.session.delete).toHaveBeenCalledTimes(2);
+  });
+});

src/tools/smartfetch/secondary-model.ts

@@ -0,0 +1,276 @@
+import { existsSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import type { PluginInput } from '@opencode-ai/plugin';
+import { stripJsonComments } from '../../cli/config-io';
+import { getConfigSearchDirs } from '../../cli/paths';
+import { loadPluginConfig } from '../../config/loader';
+import { MAX_MODEL_CONTENT_CHARS } from './constants';
+import type { CachedFetch, SecondaryModel } from './types';
+
+type OpenCodeClient = PluginInput['client'];
+
+function parseModelRef(value: string | undefined) {
+  if (!value) return undefined;
+  const [providerID, ...rest] = value.split('/');
+  const modelID = rest.join('/');
+  if (!providerID || !modelID) return undefined;
+  return { providerID, modelID };
+}
+
+function pickAgentModelRef(value: unknown): string | undefined {
+  if (typeof value === 'string') return value;
+  if (Array.isArray(value)) {
+    for (const entry of value) {
+      if (typeof entry === 'string') return entry;
+      if (
+        entry &&
+        typeof entry === 'object' &&
+        'id' in entry &&
+        typeof (entry as { id?: unknown }).id === 'string'
+      ) {
+        return (entry as { id: string }).id;
+      }
+    }
+  }
+  return undefined;
+}
+
+function findPreferredOpenCodeConfigPath(baseDir: string) {
+  for (const file of ['opencode.jsonc', 'opencode.json']) {
+    const fullPath = path.join(baseDir, file);
+    if (existsSync(fullPath)) return fullPath;
+  }
+  return undefined;
+}
+
+async function readOpenCodeConfigFile(configPath: string | undefined) {
+  if (!configPath) return undefined;
+  try {
+    const content = await readFile(configPath, 'utf8');
+    return JSON.parse(stripJsonComments(content)) as {
+      small_model?: unknown;
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+async function readEffectiveOpenCodeConfig(directory: string) {
+  const projectDir = path.join(directory, '.opencode');
+  const userDirs = getConfigSearchDirs();
+  const projectPath = findPreferredOpenCodeConfigPath(projectDir);
+  const userPath = userDirs
+    .map((configDir) => findPreferredOpenCodeConfigPath(configDir))
+    .find(Boolean);
+
+  const userConfig = await readOpenCodeConfigFile(userPath);
+  const projectConfig = await readOpenCodeConfigFile(projectPath);
+
+  return {
+    small_model: projectConfig?.small_model ?? userConfig?.small_model,
+  };
+}
+
+export async function readSecondaryModelFromConfig(directory: string) {
+  try {
+    const models: SecondaryModel[] = [];
+    const seen = new Set<string>();
+    const pushModel = (value: unknown) => {
+      if (typeof value !== 'string') return;
+      const parsedModel = parseModelRef(value);
+      if (!parsedModel) return;
+      const key = `${parsedModel.providerID}/${parsedModel.modelID}`;
+      if (seen.has(key)) return;
+      seen.add(key);
+      models.push(parsedModel);
+    };
+
+    const opencodeConfig = await readEffectiveOpenCodeConfig(directory);
+    pushModel(
+      typeof opencodeConfig.small_model === 'string'
+        ? opencodeConfig.small_model
+        : undefined,
+    );
+
+    const pluginConfig = loadPluginConfig(directory);
+    const explorerModel = pickAgentModelRef(
+      pluginConfig.agents?.explorer?.model,
+    );
+    const librarianModel = pickAgentModelRef(
+      pluginConfig.agents?.librarian?.model,
+    );
+
+    pushModel(explorerModel);
+    pushModel(librarianModel);
+
+    return models;
+  } catch {
+    return [];
+  }
+}
+
+function buildPrompt(content: string, prompt: string) {
+  return [
+    'Use only the fetched content below.',
+    'Do not use tools, outside knowledge, or unstated assumptions.',
+    'Answer concisely and directly.',
+    'If the requested information is missing from the content, say that clearly.',
+    'Preserve code examples or exact values only when they are relevant to the task.',
+    '',
+    'Fetched content:',
+    '---',
+    content,
+    '---',
+    '',
+    'Task:',
+    prompt,
+  ].join('\n');
+}
+
+export function decideSecondaryModelUse(
+  fetchResult: CachedFetch,
+  prompt: string | undefined,
+  secondaryModels: SecondaryModel[],
+) {
+  if (!prompt?.trim()) return { use: false, reason: 'no_prompt' as const };
+  if (!secondaryModels.length) {
+    return {
+      use: false,
+      reason: 'no_secondary_model_configured' as const,
+    };
+  }
+  if (!fetchResult.markdown.trim()) {
+    return { use: false, reason: 'empty_content' as const };
+  }
+  if (fetchResult.wordCount > 0 && fetchResult.wordCount < 25) {
+    return { use: false, reason: 'content_too_short' as const };
+  }
+  return { use: true, reason: 'prompt_present' as const };
+}
+
+function isUsableSecondaryText(text: string) {
+  const trimmed = text.trim();
+  if (!trimmed) return false;
+  if (/^no response from secondary model\.?$/i.test(trimmed)) return false;
+  return true;
+}
+
+async function runSecondaryModel(
+  client: OpenCodeClient,
+  directory: string,
+  model: SecondaryModel,
+  prompt: string,
+  content: string,
+) {
+  const session = await client.session.create({
+    responseStyle: 'data',
+    throwOnError: true,
+    query: { directory },
+    body: { title: 'smartfetch-secondary' },
+  });
+
+  const sessionId =
+    (session as { data?: { id?: string }; id?: string })?.data?.id ??
+    (session as { data?: { id?: string }; id?: string })?.id;
+  if (!sessionId) {
+    throw new Error('Secondary model session did not return an id');
+  }
+
+  const sourceChars = content.length;
+  const truncatedContent = content.slice(0, MAX_MODEL_CONTENT_CHARS);
+  const inputChars = truncatedContent.length;
+  const inputTruncated = inputChars < sourceChars;
+  const effectivePrompt = inputTruncated
+    ? `${prompt}\n\nNote: only the first ${inputChars} characters of a longer fetched document were provided.`
+    : prompt;
+  try {
+    const toolIDsResponse = await client.tool.ids({
+      responseStyle: 'data',
+      throwOnError: true,
+    });
+    const toolIDsData = toolIDsResponse as { data?: unknown };
+    const toolIDs = Array.isArray(toolIDsData.data)
+      ? (toolIDsData.data as string[])
+      : Array.isArray(toolIDsResponse)
+        ? toolIDsResponse
+        : [];
+    const disabledTools = Object.fromEntries(
+      (toolIDs || []).map((id: string) => [id, false]),
+    );
+
+    const result = await client.session.prompt({
+      responseStyle: 'data',
+      throwOnError: true,
+      path: { id: sessionId },
+      query: { directory },
+      body: {
+        model,
+        system:
+          'Answer only from the supplied content. Do not use tools or outside knowledge.',
+        tools: disabledTools,
+        parts: [
+          {
+            type: 'text',
+            text: buildPrompt(truncatedContent, effectivePrompt),
+          },
+        ],
+      },
+    });
+
+    const parts =
+      (result as { data?: { parts?: Array<{ type?: string; text?: string }> } })
+        ?.data?.parts ??
+      (result as { parts?: Array<{ type?: string; text?: string }> })?.parts ??
+      [];
+    const text = parts
+      .map((part) => (part?.type === 'text' ? part.text || '' : ''))
+      .join('')
+      .trim();
+
+    return {
+      text,
+      inputTruncated,
+      inputChars,
+      sourceChars,
+    };
+  } finally {
+    await client.session
+      .delete({
+        path: { id: sessionId },
+        query: { directory },
+      })
+      .catch(() => undefined);
+  }
+}
+
+export async function runSecondaryModelWithFallback(
+  client: OpenCodeClient,
+  directory: string,
+  models: SecondaryModel[],
+  prompt: string,
+  content: string,
+) {
+  let lastError: unknown;
+  for (const model of models) {
+    try {
+      const result = await runSecondaryModel(
+        client,
+        directory,
+        model,
+        prompt,
+        content,
+      );
+      if (!isUsableSecondaryText(result.text)) {
+        lastError = new Error('Secondary model returned no usable text');
+        continue;
+      }
+      return { ...result, model };
+    } catch (error) {
+      lastError = error;
+    }
+  }
+  throw lastError instanceof Error
+    ? lastError
+    : new Error(String(lastError ?? 'Secondary model failed'));
+}

src/tools/smartfetch/tool.test.ts

@@ -0,0 +1,60 @@
+import { afterEach, describe, expect, mock, test } from 'bun:test';
+import { createWebfetchTool } from './tool';
+
+function createExecutionContext() {
+  return {
+    ask: mock(async () => undefined),
+    metadata: mock(() => undefined),
+    abort: new AbortController().signal,
+    directory: '/tmp/smartfetch-test',
+  } as any;
+}
+
+describe('smartfetch/tool', () => {
+  const originalFetch = globalThis.fetch;
+
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+    mock.restore();
+  });
+
+  test('returns a required llms.txt message when prefer_llms_txt is always and no llms.txt is available', async () => {
+    const fetchMock = mock(async (input: string | URL | Request) => {
+      const url = typeof input === 'string' ? input : input.toString();
+
+      if (
+        url === 'https://docs.example.com/llms-full.txt' ||
+        url === 'https://docs.example.com/llms.txt'
+      ) {
+        return new Response('not found', {
+          status: 404,
+          headers: { 'content-type': 'text/plain' },
+        });
+      }
+
+      throw new Error(`Unexpected fetch URL: ${url}`);
+    });
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    const webfetch = createWebfetchTool({ client: {} } as any);
+    const ctx = createExecutionContext();
+    const result = await webfetch.execute(
+      {
+        url: 'https://docs.example.com/page',
+        format: 'markdown',
+        extract_main: true,
+        prefer_llms_txt: 'always',
+        include_metadata: true,
+        save_binary: false,
+      },
+      ctx,
+    );
+
+    expect(result).toContain('Required llms.txt content was unavailable.');
+    expect(result).toContain('Original URL: https://docs.example.com/page');
+    expect(result).toContain('prefer_llms_txt: "always"');
+    expect(result).toContain('used_llms_txt: false');
+    expect(ctx.ask).toHaveBeenCalledTimes(1);
+    expect(ctx.metadata).not.toHaveBeenCalled();
+  });
+});

src/tools/smartfetch/tool.ts

@@ -0,0 +1,821 @@
+import os from 'node:os';
+import path from 'node:path';
+import {
+  type PluginInput,
+  type ToolDefinition,
+  tool,
+} from '@opencode-ai/plugin';
+import { buildBinaryResultMessage, saveBinary } from './binary';
+import {
+  buildCacheKey,
+  CACHE,
+  cacheFetchResult,
+  isInvalidLlmsResult,
+} from './cache';
+import {
+  DEFAULT_TIMEOUT_SECONDS,
+  MAX_BINARY_DOWNLOAD_BYTES,
+  MAX_LLMS_PROBE_TIMEOUT_MS,
+  MAX_RESPONSE_BYTES,
+  MAX_TIMEOUT_SECONDS,
+  WEBFETCH_DESCRIPTION,
+} from './constants';
+import {
+  buildAllowedOrigins,
+  buildConditionalHeaders,
+  buildPermissionPatterns,
+  decodeBody,
+  extractHeaderMetadata,
+  fetchWithUpgradeFallback,
+  getBinaryKind,
+  isBinaryContentType,
+  isDocsLikeUrl,
+  isGenericBinaryMime,
+  isHtmlLikeContentType,
+  looksLikeHtmlText,
+  looksLikeTextBody,
+  normalizeUrl,
+  probeLlmsText,
+  readBodyLimited,
+  runWithScopedTimeout,
+} from './network';
+import {
+  decideSecondaryModelUse,
+  readSecondaryModelFromConfig,
+  runSecondaryModelWithFallback,
+} from './secondary-model';
+import type { SmartfetchOptions } from './types';
+import {
+  buildLlmsRequiredMessage,
+  buildRedirectResultMessage,
+  cleanFetchedText,
+  detectQualitySignals,
+  escapeHtml,
+  extractFromHtml,
+  extractHeadingsFromMarkdown,
+  frontmatter,
+  inferCanonicalUrlFromText,
+  joinRenderedContent,
+  pickContent,
+  renderMessageForFormat,
+  trimBlankRuns,
+  withTruncationMarker,
+  wordCount,
+} from './utils';
+
+const z = tool.schema;
+
+export function createWebfetchTool(
+  pluginCtx: PluginInput,
+  options: SmartfetchOptions = {},
+): ToolDefinition {
+  const binaryDir =
+    options.binaryDir || path.join(os.tmpdir(), 'opencode-smartfetch');
+
+  return tool({
+    description: WEBFETCH_DESCRIPTION,
+    args: {
+      url: z.httpUrl(),
+      format: z.enum(['text', 'markdown', 'html']).default('markdown'),
+      timeout: z
+        .number()
+        .positive()
+        .max(MAX_TIMEOUT_SECONDS)
+        .optional()
+        .describe('Timeout in seconds, max 120.'),
+      prompt: z
+        .string()
+        .optional()
+        .describe(
+          'Optional extraction task to run on the fetched content using a cheap secondary model.',
+        ),
+      extract_main: z.boolean().default(true),
+      prefer_llms_txt: z.enum(['auto', 'always', 'never']).default('auto'),
+      include_metadata: z.boolean().default(true),
+      save_binary: z
+        .boolean()
+        .default(false)
+        .describe(
+          'Save binary payload to disk when it fits within the active download limit.',
+        ),
+    },
+    async execute(args, ctx) {
+      const secondaryModels = await readSecondaryModelFromConfig(
+        ctx.directory || pluginCtx.directory,
+      );
+      const normalized = normalizeUrl(args.url);
+      const url = new URL(normalized.url);
+      const cacheKey = buildCacheKey(
+        args.url,
+        args.extract_main,
+        args.prefer_llms_txt,
+        args.save_binary,
+      );
+      const shouldProbeLlmsTxt =
+        args.prefer_llms_txt === 'always' ||
+        (args.prefer_llms_txt === 'auto' && isDocsLikeUrl(url));
+      const permissionPatterns = buildPermissionPatterns(
+        normalized,
+        shouldProbeLlmsTxt,
+      );
+      const allowedOrigins = buildAllowedOrigins(permissionPatterns);
+
+      await ctx.ask({
+        permission: 'webfetch',
+        patterns: permissionPatterns,
+        always: permissionPatterns,
+        metadata: {
+          url: normalized.url,
+          requested_url: args.url,
+          fallback_url: normalized.fallbackUrl,
+          llms_probe_enabled: shouldProbeLlmsTxt,
+          format: args.format,
+          prompt: args.prompt,
+        },
+      });
+
+      const timeoutMs = Math.min(
+        (args.timeout ?? DEFAULT_TIMEOUT_SECONDS) * 1000,
+        MAX_TIMEOUT_SECONDS * 1000,
+      );
+      const controller = new AbortController();
+      const timeout = setTimeout(
+        () => controller.abort(new Error(`timeout after ${timeoutMs}ms`)),
+        timeoutMs,
+      );
+      const abortHandler = () => controller.abort(ctx.abort.reason);
+      ctx.abort.addEventListener('abort', abortHandler, { once: true });
+
+      try {
+        let fetchResult = CACHE.get(cacheKey);
+        if (isInvalidLlmsResult(fetchResult)) {
+          CACHE.delete(cacheKey);
+          fetchResult = undefined;
+        }
+        const cacheHit = !!fetchResult;
+        if (fetchResult) {
+          fetchResult = {
+            ...fetchResult,
+            requestedUrl: args.url,
+            cacheHit: true,
+          };
+        }
+        if (!fetchResult) {
+          let staleFetchResult = CACHE.get(cacheKey, {
+            allowStale: true,
+            noDeleteOnStaleGet: true,
+          });
+          if (isInvalidLlmsResult(staleFetchResult)) {
+            CACHE.delete(cacheKey);
+            staleFetchResult = undefined;
+          }
+          let llmsProbeError: string | undefined;
+
+          if (shouldProbeLlmsTxt) {
+            const fallbackOrigin = normalized.fallbackUrl
+              ? new URL(normalized.fallbackUrl).origin
+              : undefined;
+            const probeTimeoutMs = Math.max(
+              1,
+              Math.min(MAX_LLMS_PROBE_TIMEOUT_MS, timeoutMs),
+            );
+            const llms = await runWithScopedTimeout(
+              controller.signal,
+              probeTimeoutMs,
+              (probeSignal) =>
+                probeLlmsText(url, timeoutMs, probeSignal, fallbackOrigin),
+            );
+            if (llms && 'text' in llms) {
+              const llmsHeaders = llms.headers || {};
+              const text = trimBlankRuns(llms.text);
+              fetchResult = {
+                requestedUrl: args.url,
+                finalUrl: llms.url,
+                statusCode: llms.statusCode,
+                contentType: llmsHeaders.contentType || 'text/plain',
+                charset: llmsHeaders.charset,
+                etag: llmsHeaders.etag,
+                lastModified: llmsHeaders.lastModified,
+                contentLength: llmsHeaders.contentLength,
+                filename: llmsHeaders.filename,
+                canonicalUrl: inferCanonicalUrlFromText(text, llms.url),
+                headings: extractHeadingsFromMarkdown(text),
+                title: undefined,
+                rawContent: text,
+                markdown: text,
+                text,
+                html: text,
+                extractedMain: false,
+                usedLlmsTxt: true,
+                sourceKind: 'llms_txt',
+                upgradedToHttps: !!llms.upgradedToHttps,
+                redirectChain: llms.redirectChain || [],
+                truncated: !!llms.truncated,
+                wordCount: wordCount(text),
+                qualitySignals: detectQualitySignals({
+                  text,
+                  markdown: text,
+                  rawContent: text,
+                  wordCount: wordCount(text),
+                  sourceKind: 'llms_txt',
+                  extractedMain: false,
+                }),
+                llmsProbeTruncated: !!llms.truncated,
+                decodedCharset: llms.decodedCharset,
+                decodeFallback: llms.decodeFallback,
+                decodeWarning: llms.decodeWarning,
+                cacheHit: false,
+              };
+              cacheFetchResult(
+                fetchResult,
+                args.extract_main,
+                args.prefer_llms_txt,
+                args.save_binary,
+              );
+            } else if (llms?.error) {
+              llmsProbeError = llms.error;
+            }
+
+            if (!fetchResult && args.prefer_llms_txt === 'always') {
+              const metadata = args.include_metadata
+                ? frontmatter({
+                    requested_url: args.url,
+                    used_llms_txt: false,
+                    llms_probe_error: llmsProbeError,
+                    prefer_llms_txt: args.prefer_llms_txt,
+                  })
+                : '';
+              return joinRenderedContent(
+                metadata,
+                renderMessageForFormat(
+                  buildLlmsRequiredMessage(args.url, llmsProbeError),
+                  args.format,
+                ),
+                args.format,
+              );
+            }
+          }
+
+          if (!fetchResult) {
+            const { result, upgradedToHttps } = await fetchWithUpgradeFallback(
+              normalized,
+              timeoutMs,
+              args.format,
+              controller.signal,
+              buildConditionalHeaders(staleFetchResult),
+              'GET',
+              allowedOrigins,
+            );
+            if ('blockedRedirect' in result) {
+              const metadata = args.include_metadata
+                ? frontmatter({
+                    requested_url: args.url,
+                    redirect_url: result.redirectUrl,
+                    status_code: result.statusCode,
+                    redirect_chain: result.redirectChain.map(
+                      (step) => `${step.status} ${step.from} -> ${step.to}`,
+                    ),
+                    upgraded_to_https: upgradedToHttps,
+                  })
+                : '';
+              return joinRenderedContent(
+                metadata,
+                renderMessageForFormat(
+                  buildRedirectResultMessage(
+                    args.url,
+                    result.redirectUrl,
+                    result.statusCode,
+                  ),
+                  args.format,
+                ),
+                args.format,
+              );
+            }
+
+            const { response, finalUrl, redirectChain } = result;
+            if (response.status === 304 && staleFetchResult) {
+              fetchResult = {
+                ...staleFetchResult,
+                requestedUrl: args.url,
+                finalUrl,
+                statusCode: staleFetchResult.statusCode,
+                redirectChain,
+                llmsProbeError,
+                cacheRevalidated: true,
+                upstreamStatusCode: 304,
+                cacheHit: false,
+              };
+              cacheFetchResult(
+                fetchResult,
+                args.extract_main,
+                args.prefer_llms_txt,
+                args.save_binary,
+              );
+            } else {
+              if (!response.ok) {
+                throw new Error(
+                  `Request failed with status code: ${response.status}`,
+                );
+              }
+              const headerMetadata = extractHeaderMetadata(
+                response.headers,
+                finalUrl,
+              );
+              const explicitBinary = isBinaryContentType(
+                headerMetadata.contentType || '',
+              );
+              const genericBinaryMime = isGenericBinaryMime(
+                headerMetadata.contentType || '',
+              );
+              const binaryDownloadLimit = args.save_binary
+                ? MAX_RESPONSE_BYTES
+                : MAX_BINARY_DOWNLOAD_BYTES;
+              if (
+                explicitBinary &&
+                !genericBinaryMime &&
+                typeof headerMetadata.contentLength === 'number' &&
+                headerMetadata.contentLength > binaryDownloadLimit
+              ) {
+                try {
+                  await response.body?.cancel();
+                } catch {
+                  // ignore cancel failures
+                }
+                fetchResult = {
+                  requestedUrl: args.url,
+                  finalUrl,
+                  statusCode: response.status,
+                  contentType:
+                    headerMetadata.contentType || 'application/octet-stream',
+                  charset: headerMetadata.charset,
+                  etag: headerMetadata.etag,
+                  lastModified: headerMetadata.lastModified,
+                  contentLength: headerMetadata.contentLength,
+                  filename: headerMetadata.filename,
+                  canonicalUrl: finalUrl,
+                  redirectChain,
+                  upgradedToHttps,
+                  truncated: false,
+                  binary: true,
+                  binaryKind: getBinaryKind(
+                    headerMetadata.contentType || 'application/octet-stream',
+                  ),
+                  downloadLimitBytes: binaryDownloadLimit,
+                  metadataOnly: true,
+                  data: undefined,
+                  llmsProbeError,
+                  llmsProbeTruncated: false,
+                  cacheHit: false,
+                };
+                cacheFetchResult(
+                  fetchResult,
+                  args.extract_main,
+                  args.prefer_llms_txt,
+                  args.save_binary,
+                );
+              } else {
+                const readLimit =
+                  explicitBinary && !genericBinaryMime
+                    ? binaryDownloadLimit
+                    : MAX_RESPONSE_BYTES;
+                const body = await readBodyLimited(response, readLimit);
+                const provisionalDecoded =
+                  !headerMetadata.contentType ||
+                  genericBinaryMime ||
+                  /^text\//i.test(headerMetadata.contentType)
+                    ? decodeBody(
+                        body.data,
+                        headerMetadata.charset,
+                        headerMetadata.contentType,
+                      )
+                    : undefined;
+                const looksHtmlPayload = provisionalDecoded
+                  ? looksLikeHtmlText(provisionalDecoded.text)
+                  : false;
+                const contentType = headerMetadata.contentType
+                  ? genericBinaryMime && looksLikeTextBody(body.data)
+                    ? looksHtmlPayload
+                      ? 'text/html'
+                      : 'text/plain'
+                    : /^text\/plain(?:;|$)/i.test(headerMetadata.contentType) &&
+                        looksHtmlPayload
+                      ? 'text/html'
+                      : headerMetadata.contentType
+                  : looksLikeTextBody(body.data)
+                    ? looksHtmlPayload
+                      ? 'text/html'
+                      : 'text/plain'
+                    : 'application/octet-stream';
+
+                if (isBinaryContentType(contentType)) {
+                  const binaryTooLarge =
+                    body.truncated ||
+                    (typeof headerMetadata.contentLength === 'number' &&
+                      headerMetadata.contentLength > binaryDownloadLimit);
+                  fetchResult = {
+                    requestedUrl: args.url,
+                    finalUrl,
+                    statusCode: response.status,
+                    contentType,
+                    charset: headerMetadata.charset,
+                    etag: headerMetadata.etag,
+                    lastModified: headerMetadata.lastModified,
+                    contentLength: headerMetadata.contentLength,
+                    filename: headerMetadata.filename,
+                    canonicalUrl: finalUrl,
+                    redirectChain,
+                    upgradedToHttps,
+                    truncated: body.truncated,
+                    binary: true,
+                    binaryKind: getBinaryKind(contentType),
+                    downloadLimitBytes: binaryDownloadLimit,
+                    metadataOnly: binaryTooLarge,
+                    data: binaryTooLarge ? undefined : body.data,
+                    llmsProbeError,
+                    llmsProbeTruncated: false,
+                    cacheHit: false,
+                  };
+                  cacheFetchResult(
+                    fetchResult,
+                    args.extract_main,
+                    args.prefer_llms_txt,
+                    args.save_binary,
+                  );
+                } else {
+                  const decoded =
+                    provisionalDecoded ||
+                    decodeBody(body.data, headerMetadata.charset, contentType);
+                  const rawText = decoded.text;
+                  const extracted = isHtmlLikeContentType(contentType)
+                    ? extractFromHtml(rawText, finalUrl, args.extract_main)
+                    : {
+                        title: undefined,
+                        rawContent: cleanFetchedText(rawText),
+                        html: cleanFetchedText(rawText),
+                        text: cleanFetchedText(rawText),
+                        markdown: cleanFetchedText(rawText),
+                        extractedMain: false,
+                        canonicalUrl: undefined,
+                        headings: [],
+                      };
+
+                  fetchResult = {
+                    requestedUrl: args.url,
+                    finalUrl,
+                    statusCode: response.status,
+                    contentType,
+                    charset: headerMetadata.charset,
+                    etag: headerMetadata.etag,
+                    lastModified: headerMetadata.lastModified,
+                    contentLength: headerMetadata.contentLength,
+                    filename: headerMetadata.filename,
+                    canonicalUrl:
+                      extracted.canonicalUrl ||
+                      inferCanonicalUrlFromText(extracted.markdown, finalUrl) ||
+                      finalUrl,
+                    headings: extracted.headings?.length
+                      ? extracted.headings
+                      : extractHeadingsFromMarkdown(extracted.markdown),
+                    title: extracted.title,
+                    rawContent: extracted.rawContent,
+                    markdown: extracted.markdown,
+                    text: extracted.text,
+                    html: extracted.html,
+                    extractedMain: extracted.extractedMain,
+                    usedLlmsTxt: false,
+                    sourceKind: isHtmlLikeContentType(contentType)
+                      ? 'html'
+                      : 'text',
+                    upgradedToHttps,
+                    redirectChain,
+                    truncated: body.truncated,
+                    wordCount: wordCount(extracted.text),
+                    qualitySignals: detectQualitySignals({
+                      text: extracted.text,
+                      markdown: extracted.markdown,
+                      rawContent: extracted.rawContent,
+                      wordCount: wordCount(extracted.text),
+                      sourceKind: isHtmlLikeContentType(contentType)
+                        ? 'html'
+                        : 'text',
+                      extractedMain: extracted.extractedMain,
+                    }),
+                    llmsProbeError,
+                    llmsProbeTruncated: false,
+                    decodedCharset: decoded.decodedCharset,
+                    decodeFallback: decoded.decodeFallback,
+                    decodeWarning: decoded.decodeWarning,
+                    cacheHit: false,
+                  };
+                  cacheFetchResult(
+                    fetchResult,
+                    args.extract_main,
+                    args.prefer_llms_txt,
+                    args.save_binary,
+                  );
+                }
+              }
+            }
+          }
+        }
+
+        ctx.metadata({
+          title:
+            ('binary' in fetchResult
+              ? fetchResult.filename
+              : fetchResult.title) || fetchResult.finalUrl,
+          metadata: {
+            url: fetchResult.finalUrl,
+            contentType: fetchResult.contentType,
+            truncated: fetchResult.truncated,
+          },
+        });
+
+        if ('binary' in fetchResult) {
+          if (fetchResult.metadataOnly || !fetchResult.data) {
+            const metadata = args.include_metadata
+              ? frontmatter({
+                  requested_url: fetchResult.requestedUrl,
+                  final_url: fetchResult.finalUrl,
+                  canonical_url: fetchResult.canonicalUrl,
+                  status_code: fetchResult.statusCode,
+                  source_content_type: fetchResult.contentType,
+                  charset: fetchResult.charset,
+                  etag: fetchResult.etag,
+                  last_modified: fetchResult.lastModified,
+                  content_length: fetchResult.contentLength,
+                  filename: fetchResult.filename,
+                  binary_kind: fetchResult.binaryKind,
+                  redirect_chain: fetchResult.redirectChain.map(
+                    (step) => `${step.status} ${step.from} -> ${step.to}`,
+                  ),
+                  upgraded_to_https: fetchResult.upgradedToHttps,
+                  llms_probe_error: fetchResult.llmsProbeError,
+                  cache_revalidated: fetchResult.cacheRevalidated,
+                  cache_hit: fetchResult.cacheHit ?? cacheHit,
+                  upstream_status_code: fetchResult.upstreamStatusCode,
+                  truncated: fetchResult.truncated,
+                  download_limit_bytes:
+                    fetchResult.downloadLimitBytes ?? MAX_BINARY_DOWNLOAD_BYTES,
+                  binary_metadata_only: true,
+                })
+              : '';
+            return joinRenderedContent(
+              metadata,
+              renderMessageForFormat(
+                buildBinaryResultMessage(fetchResult),
+                args.format,
+              ),
+              args.format,
+            );
+          }
+          if (!args.save_binary) {
+            const metadata = args.include_metadata
+              ? frontmatter({
+                  requested_url: fetchResult.requestedUrl,
+                  final_url: fetchResult.finalUrl,
+                  canonical_url: fetchResult.canonicalUrl,
+                  status_code: fetchResult.statusCode,
+                  source_content_type: fetchResult.contentType,
+                  charset: fetchResult.charset,
+                  etag: fetchResult.etag,
+                  last_modified: fetchResult.lastModified,
+                  content_length: fetchResult.contentLength,
+                  filename: fetchResult.filename,
+                  binary_kind: fetchResult.binaryKind,
+                  redirect_chain: fetchResult.redirectChain.map(
+                    (step) => `${step.status} ${step.from} -> ${step.to}`,
+                  ),
+                  upgraded_to_https: fetchResult.upgradedToHttps,
+                  truncated: fetchResult.truncated,
+                  save_binary: false,
+                  cache_hit: fetchResult.cacheHit ?? cacheHit,
+                })
+              : '';
+            return joinRenderedContent(
+              metadata,
+              renderMessageForFormat(
+                `${fetchResult.binaryKind.toUpperCase()} content fetched but not saved. Re-run with save_binary=true to persist it.`,
+                args.format,
+              ),
+              args.format,
+            );
+          }
+          const savedPath = await saveBinary(
+            binaryDir,
+            fetchResult.data,
+            fetchResult.contentType,
+            fetchResult.filename,
+          );
+          const metadata = args.include_metadata
+            ? frontmatter({
+                requested_url: fetchResult.requestedUrl,
+                final_url: fetchResult.finalUrl,
+                canonical_url: fetchResult.canonicalUrl,
+                status_code: fetchResult.statusCode,
+                source_content_type: fetchResult.contentType,
+                charset: fetchResult.charset,
+                etag: fetchResult.etag,
+                last_modified: fetchResult.lastModified,
+                content_length: fetchResult.contentLength,
+                filename: fetchResult.filename,
+                binary_kind: fetchResult.binaryKind,
+                redirect_chain: fetchResult.redirectChain.map(
+                  (step) => `${step.status} ${step.from} -> ${step.to}`,
+                ),
+                upgraded_to_https: fetchResult.upgradedToHttps,
+                llms_probe_error: fetchResult.llmsProbeError,
+                cache_revalidated: fetchResult.cacheRevalidated,
+                cache_hit: fetchResult.cacheHit ?? cacheHit,
+                upstream_status_code: fetchResult.upstreamStatusCode,
+                truncated: fetchResult.truncated,
+                download_limit_bytes:
+                  fetchResult.downloadLimitBytes ?? MAX_BINARY_DOWNLOAD_BYTES,
+                saved_path: savedPath,
+              })
+            : '';
+          return joinRenderedContent(
+            metadata,
+            renderMessageForFormat(
+              buildBinaryResultMessage(fetchResult, savedPath),
+              args.format,
+            ),
+            args.format,
+          );
+        }
+
+        const baseContent = pickContent(fetchResult, args.format);
+        const secondaryModelDecision = decideSecondaryModelUse(
+          fetchResult,
+          args.prompt,
+          secondaryModels,
+        );
+        const metadata = args.include_metadata
+          ? frontmatter({
+              requested_url: fetchResult.requestedUrl,
+              final_url: fetchResult.finalUrl,
+              canonical_url: fetchResult.canonicalUrl,
+              status_code: fetchResult.statusCode,
+              source_content_type: fetchResult.contentType,
+              charset: fetchResult.charset,
+              etag: fetchResult.etag,
+              last_modified: fetchResult.lastModified,
+              content_length: fetchResult.contentLength,
+              filename: fetchResult.filename,
+              headings: fetchResult.headings,
+              title: fetchResult.title,
+              source_kind: fetchResult.sourceKind,
+              used_llms_txt: fetchResult.usedLlmsTxt,
+              extracted_main: fetchResult.extractedMain,
+              redirect_chain: fetchResult.redirectChain.map(
+                (step) => `${step.status} ${step.from} -> ${step.to}`,
+              ),
+              upgraded_to_https: fetchResult.upgradedToHttps,
+              llms_probe_error: fetchResult.llmsProbeError,
+              llms_probe_truncated: fetchResult.llmsProbeTruncated,
+              cache_revalidated: fetchResult.cacheRevalidated,
+              cache_hit: fetchResult.cacheHit ?? cacheHit,
+              upstream_status_code: fetchResult.upstreamStatusCode,
+              truncated: fetchResult.truncated,
+              word_count: fetchResult.wordCount,
+              quality_signals: fetchResult.qualitySignals,
+              decoded_charset: fetchResult.decodedCharset,
+              decode_fallback: fetchResult.decodeFallback,
+              decode_warning: fetchResult.decodeWarning,
+              secondary_model: undefined,
+              secondary_model_skipped_reason:
+                !secondaryModelDecision.use && args.prompt
+                  ? secondaryModelDecision.reason
+                  : undefined,
+            })
+          : '';
+
+        if (!secondaryModelDecision.use) {
+          return joinRenderedContent(metadata, baseContent, args.format);
+        }
+
+        if (!secondaryModels.length) {
+          return joinRenderedContent(metadata, baseContent, args.format);
+        }
+        let secondaryRun:
+          | Awaited<ReturnType<typeof runSecondaryModelWithFallback>>
+          | undefined;
+        let secondaryModelError: string | undefined;
+        try {
+          secondaryRun = await runSecondaryModelWithFallback(
+            pluginCtx.client,
+            ctx.directory || process.cwd(),
+            secondaryModels,
+            args.prompt || '',
+            fetchResult.markdown,
+          );
+        } catch (error: unknown) {
+          secondaryModelError =
+            error instanceof Error ? error.message : String(error);
+        }
+
+        if (!secondaryRun) {
+          const degradedMetadata = args.include_metadata
+            ? frontmatter({
+                requested_url: fetchResult.requestedUrl,
+                final_url: fetchResult.finalUrl,
+                canonical_url: fetchResult.canonicalUrl,
+                status_code: fetchResult.statusCode,
+                source_content_type: fetchResult.contentType,
+                charset: fetchResult.charset,
+                etag: fetchResult.etag,
+                last_modified: fetchResult.lastModified,
+                content_length: fetchResult.contentLength,
+                filename: fetchResult.filename,
+                headings: fetchResult.headings,
+                title: fetchResult.title,
+                source_kind: fetchResult.sourceKind,
+                used_llms_txt: fetchResult.usedLlmsTxt,
+                extracted_main: fetchResult.extractedMain,
+                redirect_chain: fetchResult.redirectChain.map(
+                  (step) => `${step.status} ${step.from} -> ${step.to}`,
+                ),
+                upgraded_to_https: fetchResult.upgradedToHttps,
+                llms_probe_error: fetchResult.llmsProbeError,
+                llms_probe_truncated: fetchResult.llmsProbeTruncated,
+                cache_revalidated: fetchResult.cacheRevalidated,
+                cache_hit: fetchResult.cacheHit ?? cacheHit,
+                upstream_status_code: fetchResult.upstreamStatusCode,
+                truncated: fetchResult.truncated,
+                word_count: fetchResult.wordCount,
+                quality_signals: fetchResult.qualitySignals,
+                decoded_charset: fetchResult.decodedCharset,
+                decode_fallback: fetchResult.decodeFallback,
+                decode_warning: fetchResult.decodeWarning,
+                secondary_model: undefined,
+                secondary_model_skipped_reason: 'secondary_model_failed',
+                secondary_model_error: secondaryModelError,
+              })
+            : '';
+          return joinRenderedContent(
+            degradedMetadata,
+            baseContent,
+            args.format,
+          );
+        }
+
+        const metadataWithSecondary = args.include_metadata
+          ? frontmatter({
+              requested_url: fetchResult.requestedUrl,
+              final_url: fetchResult.finalUrl,
+              canonical_url: fetchResult.canonicalUrl,
+              status_code: fetchResult.statusCode,
+              source_content_type: fetchResult.contentType,
+              charset: fetchResult.charset,
+              etag: fetchResult.etag,
+              last_modified: fetchResult.lastModified,
+              content_length: fetchResult.contentLength,
+              filename: fetchResult.filename,
+              headings: fetchResult.headings,
+              title: fetchResult.title,
+              source_kind: fetchResult.sourceKind,
+              used_llms_txt: fetchResult.usedLlmsTxt,
+              extracted_main: fetchResult.extractedMain,
+              redirect_chain: fetchResult.redirectChain.map(
+                (step) => `${step.status} ${step.from} -> ${step.to}`,
+              ),
+              upgraded_to_https: fetchResult.upgradedToHttps,
+              llms_probe_error: fetchResult.llmsProbeError,
+              llms_probe_truncated: fetchResult.llmsProbeTruncated,
+              cache_revalidated: fetchResult.cacheRevalidated,
+              cache_hit: fetchResult.cacheHit ?? cacheHit,
+              upstream_status_code: fetchResult.upstreamStatusCode,
+              truncated: fetchResult.truncated,
+              word_count: fetchResult.wordCount,
+              quality_signals: fetchResult.qualitySignals,
+              decoded_charset: fetchResult.decodedCharset,
+              decode_fallback: fetchResult.decodeFallback,
+              decode_warning: fetchResult.decodeWarning,
+              secondary_model_input_truncated: secondaryRun.inputTruncated,
+              secondary_model_input_chars: secondaryRun.inputChars,
+              secondary_model_source_chars: secondaryRun.sourceChars,
+              secondary_model: `${secondaryRun.model.providerID}/${secondaryRun.model.modelID}`,
+            })
+          : '';
+        const secondaryRaw =
+          secondaryRun.text || 'No response from secondary model.';
+        const secondaryContent =
+          args.format === 'html'
+            ? withTruncationMarker(
+                `<pre>${escapeHtml(secondaryRaw)}</pre>`,
+                'html',
+                false,
+              )
+            : withTruncationMarker(secondaryRaw, args.format, false);
+        return joinRenderedContent(
+          metadataWithSecondary,
+          secondaryContent,
+          args.format,
+        );
+      } finally {
+        clearTimeout(timeout);
+        ctx.abort.removeEventListener('abort', abortHandler);
+      }
+    },
+  });
+}

src/tools/smartfetch/types.ts

@@ -0,0 +1,135 @@
+export type SmartfetchOptions = {
+  binaryDir?: string;
+};
+
+export type SecondaryModel = {
+  providerID: string;
+  modelID: string;
+};
+
+export type RedirectStep = {
+  from: string;
+  to: string;
+  status: number;
+};
+
+export type CachedFetch = {
+  requestedUrl: string;
+  finalUrl: string;
+  statusCode: number;
+  contentType: string;
+  charset?: string;
+  etag?: string;
+  lastModified?: string;
+  contentLength?: number;
+  filename?: string;
+  canonicalUrl?: string;
+  headings?: string[];
+  title?: string;
+  rawContent: string;
+  markdown: string;
+  text: string;
+  html: string;
+  extractedMain: boolean;
+  usedLlmsTxt: boolean;
+  sourceKind: 'llms_txt' | 'html' | 'text';
+  upgradedToHttps: boolean;
+  redirectChain: RedirectStep[];
+  truncated: boolean;
+  wordCount: number;
+  qualitySignals?: string[];
+  llmsProbeError?: string;
+  llmsProbeTruncated?: boolean;
+  cacheRevalidated?: boolean;
+  upstreamStatusCode?: number;
+  cacheHit?: boolean;
+  decodedCharset?: string;
+  decodeFallback?: boolean;
+  decodeWarning?: string;
+  secondaryModelInputTruncated?: boolean;
+  secondaryModelInputChars?: number;
+  secondaryModelSourceChars?: number;
+};
+
+export type BinaryFetch = {
+  requestedUrl: string;
+  finalUrl: string;
+  statusCode: number;
+  contentType: string;
+  charset?: string;
+  etag?: string;
+  lastModified?: string;
+  contentLength?: number;
+  filename?: string;
+  canonicalUrl?: string;
+  redirectChain: RedirectStep[];
+  upgradedToHttps: boolean;
+  truncated: boolean;
+  binary: true;
+  binaryKind: 'image' | 'audio' | 'video' | 'pdf' | 'binary';
+  downloadLimitBytes?: number;
+  metadataOnly?: boolean;
+  data?: Uint8Array;
+  llmsProbeError?: string;
+  llmsProbeTruncated?: boolean;
+  cacheRevalidated?: boolean;
+  upstreamStatusCode?: number;
+  cacheHit?: boolean;
+};
+
+export type FetchResult = CachedFetch | BinaryFetch;
+
+export type DecodedBody = {
+  text: string;
+  decodedCharset: string;
+  decodeFallback: boolean;
+  decodeWarning?: string;
+};
+
+export type ExtractedContent = {
+  title?: string;
+  rawContent: string;
+  markdown: string;
+  text: string;
+  html: string;
+  extractedMain: boolean;
+  canonicalUrl?: string;
+  headings?: string[];
+};
+
+export type FetchWithRedirectsResult =
+  | {
+      blockedRedirect: true;
+      redirectUrl: string;
+      statusCode: number;
+      redirectChain: RedirectStep[];
+    }
+  | {
+      response: Response;
+      finalUrl: string;
+      redirectChain: RedirectStep[];
+    };
+
+export type LlmsProbeResult =
+  | {
+      url: string;
+      statusCode: number;
+      redirectChain: RedirectStep[];
+      text: string;
+      headers: {
+        contentType?: string;
+        charset?: string;
+        etag?: string;
+        lastModified?: string;
+        contentLength?: number;
+        filename?: string;
+      };
+      truncated: boolean;
+      decodedCharset: string;
+      decodeFallback: boolean;
+      decodeWarning?: string;
+      upgradedToHttps: boolean;
+    }
+  | {
+      error?: string;
+    };

src/tools/smartfetch/utils.test.ts

@@ -0,0 +1,24 @@
+import { describe, expect, test } from 'bun:test';
+import { extractHeadingsFromMarkdown, joinRenderedContent } from './utils';
+
+describe('smartfetch/utils', () => {
+  test('extracts cleaned headings from markdown', () => {
+    const headings = extractHeadingsFromMarkdown(
+      ['# Intro', '## Details ###', '### C#', 'plain text'].join('\n'),
+    );
+
+    expect(headings).toEqual(['Intro', 'Details', 'C#']);
+  });
+
+  test('injects metadata comments after an XML declaration in html output', () => {
+    const result = joinRenderedContent(
+      '---\nsource: "smartfetch"\n---\n\n',
+      '<?xml version="1.0"?><root>ok</root>',
+      'html',
+    );
+
+    expect(result).toStartWith('<?xml version="1.0"?>');
+    expect(result).toContain('<!--\n---\nsource: "smartfetch"\n---\n-->');
+    expect(result).toContain('<root>ok</root>');
+  });
+});

src/tools/smartfetch/utils.ts

@@ -0,0 +1,443 @@
+import { Readability } from '@mozilla/readability';
+import { JSDOM } from 'jsdom';
+import TurndownService from 'turndown';
+import type { CachedFetch, ExtractedContent } from './types';
+
+export function wordCount(text: string) {
+  const trimmed = text.trim();
+  if (!trimmed) return 0;
+  return trimmed.split(/\s+/).length;
+}
+
+function byteLength(text: string) {
+  return Buffer.byteLength(text || '', 'utf8');
+}
+
+function quote(value: unknown) {
+  return JSON.stringify(value ?? '');
+}
+
+export function frontmatter(metadata: Record<string, unknown>) {
+  const lines = ['---'];
+  for (const [key, value] of Object.entries(metadata)) {
+    if (value === undefined) continue;
+    if (Array.isArray(value)) {
+      if (value.length === 0) {
+        lines.push(`${key}: []`);
+        continue;
+      }
+      lines.push(`${key}:`);
+      for (const item of value) lines.push(`  - ${quote(item)}`);
+      continue;
+    }
+    lines.push(`${key}: ${quote(value)}`);
+  }
+  lines.push('---', '', '');
+  return lines.join('\n');
+}
+
+export function trimBlankRuns(input: string) {
+  return input.replace(/\n{3,}/g, '\n\n').trim();
+}
+
+function cleanExtractedText(input: string) {
+  return trimBlankRuns(input);
+}
+
+function mapOutsideCodeBlocks(
+  input: string,
+  transform: (value: string) => string,
+) {
+  const parts = input.split(/(```[\s\S]*?```|~~~[\s\S]*?~~~)/g);
+  return parts
+    .map((part, index) => (index % 2 === 1 ? part : transform(part)))
+    .join('');
+}
+
+function extractStructuredText(root: Element | null) {
+  if (!root) return '';
+  const chunks: string[] = [];
+  const ignoredTags = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE']);
+  const blockTags = new Set([
+    'ARTICLE',
+    'ASIDE',
+    'BLOCKQUOTE',
+    'DIV',
+    'DL',
+    'DT',
+    'DD',
+    'FIGCAPTION',
+    'FIGURE',
+    'FOOTER',
+    'FORM',
+    'H1',
+    'H2',
+    'H3',
+    'H4',
+    'H5',
+    'H6',
+    'HEADER',
+    'HR',
+    'LI',
+    'MAIN',
+    'NAV',
+    'OL',
+    'P',
+    'PRE',
+    'SECTION',
+    'TABLE',
+    'TBODY',
+    'TD',
+    'TH',
+    'THEAD',
+    'TR',
+    'UL',
+  ]);
+  const isText = (node: Node) => node.nodeType === node.TEXT_NODE;
+  const isElement = (node: Node) => node.nodeType === node.ELEMENT_NODE;
+  const pushText = (value: string) => {
+    const normalized = value.replace(/\s+/g, ' ');
+    if (!normalized.trim()) return;
+    const previous = chunks[chunks.length - 1];
+    if (!previous || /\n$| $/.test(previous)) {
+      chunks.push(normalized.trimStart());
+    } else {
+      chunks.push(normalized);
+    }
+  };
+  const pushBreak = (count = 1) => {
+    const wanted = '\n'.repeat(count);
+    const last = chunks[chunks.length - 1] || '';
+    const trailing = last.match(/\n+$/)?.[0].length || 0;
+    if (trailing >= count) return;
+    if (trailing > 0) {
+      chunks[chunks.length - 1] = last.replace(/\n+$/, '') + wanted;
+      return;
+    }
+    chunks.push(wanted);
+  };
+  const visit = (node: Node) => {
+    if (isText(node)) {
+      pushText(node.textContent || '');
+      return;
+    }
+    if (!isElement(node)) return;
+    const element = node as Element;
+    const tag = element.tagName;
+    if (ignoredTags.has(tag)) return;
+    if (tag === 'BR') {
+      pushBreak(1);
+      return;
+    }
+    if (tag === 'PRE') {
+      const text = trimBlankRuns(element.textContent || '');
+      if (!text) return;
+      pushBreak(2);
+      chunks.push(text);
+      pushBreak(2);
+      return;
+    }
+    const isBlock = blockTags.has(tag);
+    if (isBlock) pushBreak(tag === 'LI' ? 1 : 2);
+    if (tag === 'LI') chunks.push('- ');
+    for (const child of element.childNodes) visit(child);
+    if (isBlock) pushBreak(tag === 'LI' ? 1 : 2);
+  };
+  visit(root);
+  return cleanExtractedText(chunks.join(''));
+}
+
+export function cleanHeadingText(input: string) {
+  const normalized = trimBlankRuns(input).replace(/¶+$/g, '').trim();
+  if (/^(?:C|F)#$/.test(normalized)) return normalized;
+  if (/\s#+$/.test(normalized)) {
+    return normalized.replace(/\s#+$/g, '').trim();
+  }
+  return normalized;
+}
+
+export function cleanFetchedMarkdown(input: string) {
+  const output = mapOutsideCodeBlocks(input, (value) =>
+    value
+      .replace(/^\s*!\[[^\]]*\]\([^)]+\)\s*$/gm, 'Image omitted')
+      .replace(/(^|\n)Image(?=\n|$)/g, '$1Image omitted')
+      .replace(/^\s*(#{1,6})\s*\\?\['([^'\n]+)'\s*$/gm, '$1 $2')
+      .replace(/^\s*(#{1,6})\s*'([^'\n]+)'\]\s*$/gm, '$1 $2')
+      .replace(/^\s*(#{1,6})\s*'([^'\n]+)'\s*$/gm, '$1 $2')
+      .replace(/(#{1,6}[^\n]*?)\s*\[¶\]\(#.*?"Permanent link"\)\s*$/gm, '$1')
+      .replace(/\s+\(#[A-Za-z0-9_-]+\)\s*$/gm, ''),
+  );
+
+  return trimBlankRuns(output);
+}
+
+export function cleanFetchedText(input: string) {
+  return trimBlankRuns(input);
+}
+
+export function escapeHtml(input: string) {
+  return input
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&#39;');
+}
+
+export function withTruncationMarker(
+  content: string,
+  format: 'text' | 'markdown' | 'html',
+  truncated: boolean,
+) {
+  if (!truncated) return content;
+  if (format === 'html') return `${content}\n<!-- [..content truncated..] -->`;
+  return `${content}\n\n[..content truncated..]`;
+}
+
+export function joinRenderedContent(
+  metadata: string,
+  content: string,
+  format: 'text' | 'markdown' | 'html',
+) {
+  if (!metadata) return content;
+  if (!content) {
+    return format === 'html' ? `<!--\n${metadata.trim()}\n-->` : metadata;
+  }
+  if (format === 'html') {
+    const comment = `<!--\n${metadata.trim()}\n-->\n`;
+    const xmlDecl = content.match(/^\s*(<\?xml[\s\S]*?\?>\s*)/i);
+    if (xmlDecl) {
+      return `${xmlDecl[1]}${comment}${content.slice(xmlDecl[0].length)}`;
+    }
+    return `${comment}${content}`;
+  }
+  const startsWithFrontmatter = /^---(?:\r?\n|$)/.test(content);
+  if (!startsWithFrontmatter) return `${metadata}${content}`;
+  return `${metadata}Source content:\n\n${content}`;
+}
+
+export function renderMessageForFormat(
+  content: string,
+  format: 'text' | 'markdown' | 'html',
+) {
+  if (format === 'html') return `<pre>${escapeHtml(content)}</pre>`;
+  return content;
+}
+
+export function buildRedirectResultMessage(
+  originalUrl: string,
+  redirectUrl: string,
+  statusCode: number,
+) {
+  return [
+    'Redirect was blocked by policy.',
+    `Original URL: ${originalUrl}`,
+    `Redirect URL: ${redirectUrl}`,
+    `Status: ${statusCode}`,
+    '',
+    'Re-run webfetch with the redirect URL to continue.',
+  ].join('\n');
+}
+
+export function buildLlmsRequiredMessage(originalUrl: string, reason?: string) {
+  return [
+    'Required llms.txt content was unavailable.',
+    `Original URL: ${originalUrl}`,
+    ...(reason ? [`Reason: ${reason}`] : []),
+  ].join('\n');
+}
+
+const turndown = new TurndownService({
+  headingStyle: 'atx',
+  bulletListMarker: '-',
+  codeBlockStyle: 'fenced',
+});
+
+turndown.remove(['script', 'style', 'noscript', 'meta', 'link']);
+turndown.remove(
+  (node) =>
+    node.nodeName === 'A' &&
+    /permanent link/i.test((node as Element).getAttribute('title') || ''),
+);
+turndown.addRule('fenced-pre-code', {
+  filter(node) {
+    return node.nodeName === 'PRE' && !!(node as Element).querySelector('code');
+  },
+  replacement(_content, node) {
+    const code = (node as Element).querySelector('code');
+    const text = trimBlankRuns(code?.textContent || node.textContent || '');
+    if (!text) return '';
+    return `\n\n\`\`\`\n${text}\n\`\`\`\n\n`;
+  },
+});
+
+export function extractFromHtml(
+  html: string,
+  finalUrl: string,
+  extractMain: boolean,
+): ExtractedContent {
+  const dom = new JSDOM(html, { url: finalUrl });
+  const document = dom.window.document;
+  const title = document.title || undefined;
+  const canonical =
+    document.querySelector('link[rel="canonical"]')?.getAttribute('href') ||
+    undefined;
+  const canonicalUrl = (() => {
+    if (!canonical) return undefined;
+    try {
+      return new URL(canonical, finalUrl).toString();
+    } catch {
+      return undefined;
+    }
+  })();
+  const headings = Array.from(
+    document.querySelectorAll<HTMLElement>('h1, h2, h3'),
+  )
+    .map((node) => cleanHeadingText(node.textContent || ''))
+    .filter(Boolean)
+    .slice(0, 12);
+
+  if (extractMain) {
+    const readerDom = new JSDOM(html, { url: finalUrl });
+    const article = new Readability(readerDom.window.document).parse();
+    if (article?.content?.trim()) {
+      const articleContainer = readerDom.window.document.createElement('div');
+      articleContainer.innerHTML = article.content;
+      const articleText = extractStructuredText(articleContainer);
+      const articleMarkdown = trimBlankRuns(turndown.turndown(article.content));
+      return {
+        title: article.title || title,
+        rawContent: html,
+        html: article.content,
+        text: articleText,
+        markdown: articleMarkdown,
+        extractedMain: true,
+        canonicalUrl,
+        headings,
+      };
+    }
+  }
+
+  const bodyHtml = document.body?.innerHTML || html;
+  const bodyText = extractStructuredText(document.body);
+  const markdown = trimBlankRuns(turndown.turndown(bodyHtml));
+  return {
+    title,
+    rawContent: html,
+    html: bodyHtml,
+    text: bodyText,
+    markdown,
+    extractedMain: false,
+    canonicalUrl,
+    headings,
+  };
+}
+
+function parseFrontmatterBlock(content: string) {
+  const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
+  if (!match) return undefined;
+  const result: Record<string, string> = {};
+  for (const line of match[1].split(/\r?\n/)) {
+    const kv = line.match(/^([A-Za-z0-9_-]+):\s*(.+?)\s*$/);
+    if (!kv) continue;
+    result[kv[1]] = kv[2].replace(/^(['"])(.*)\1$/, '$2');
+  }
+  return result;
+}
+
+export function inferCanonicalUrlFromText(content: string, finalUrl: string) {
+  const frontmatter = parseFrontmatterBlock(content);
+  const raw = frontmatter?.url;
+  if (!raw) return undefined;
+  try {
+    return new URL(raw, finalUrl).toString();
+  } catch {
+    return undefined;
+  }
+}
+
+export function extractHeadingsFromMarkdown(content: string) {
+  const headings = content
+    .split(/\r?\n/)
+    .filter((line) => /^#{1,6}\s+/.test(line))
+    .map((line) => cleanHeadingText(line.replace(/^#{1,6}\s+/, '')))
+    .filter(Boolean)
+    .slice(0, 12);
+  return headings.length ? headings : undefined;
+}
+
+export function detectQualitySignals(
+  fetchResult: Pick<
+    CachedFetch,
+    | 'text'
+    | 'markdown'
+    | 'rawContent'
+    | 'wordCount'
+    | 'sourceKind'
+    | 'extractedMain'
+  >,
+) {
+  const signals = new Set<string>();
+  const text = `${fetchResult.text}\n${fetchResult.markdown}`.toLowerCase();
+
+  if (fetchResult.wordCount > 0 && fetchResult.wordCount < 60) {
+    signals.add('very_short_content');
+  }
+
+  if (
+    /(subscribe to continue|subscription required|sign in to continue|log in to continue|create an account to continue|members only|premium content|paywall)/i.test(
+      text,
+    )
+  ) {
+    signals.add('possible_paywall');
+  }
+
+  if (fetchResult.sourceKind === 'html') {
+    const renderedBytes = Math.max(byteLength(fetchResult.text), 1);
+    const rawBytes = byteLength(fetchResult.rawContent);
+    const ratio = rawBytes / renderedBytes;
+    if (
+      !fetchResult.extractedMain &&
+      ratio >= 10 &&
+      fetchResult.wordCount < 1200
+    ) {
+      signals.add('high_boilerplate_ratio');
+    }
+  }
+
+  return [...signals];
+}
+
+export function pickContent(
+  fetchResult: CachedFetch,
+  format: 'text' | 'markdown' | 'html',
+) {
+  if (format === 'html') {
+    if (fetchResult.sourceKind === 'html') {
+      const htmlContent = fetchResult.extractedMain
+        ? fetchResult.html
+        : fetchResult.rawContent;
+      return withTruncationMarker(htmlContent, format, fetchResult.truncated);
+    }
+    return withTruncationMarker(
+      renderMessageForFormat(
+        fetchResult.text || fetchResult.rawContent,
+        format,
+      ),
+      format,
+      fetchResult.truncated,
+    );
+  }
+  if (format === 'text') {
+    return withTruncationMarker(
+      cleanFetchedText(fetchResult.text),
+      format,
+      fetchResult.truncated,
+    );
+  }
+  return withTruncationMarker(
+    cleanFetchedMarkdown(fetchResult.markdown),
+    format,
+    fetchResult.truncated,
+  );
+}