diff --git a/apps/api/package.json b/apps/api/package.json index a2920c5e0d..b8cd62c33a 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -111,7 +111,7 @@ "culori": "^4.0.2", "dotenv": "^16.3.1", "drizzle-orm": "1.0.0-rc.3", - "esbuild": "^0.27.2", + "esbuild": "^0.28.1", "escape-html": "^1.0.3", "express": "4.22.0", "express-ws": "^5.0.2", @@ -208,7 +208,8 @@ "follow-redirects@<1.16.0": ">=1.16.0 <2.0.0", "uuid@<11.1.1": "11.1.1", "js-cookie@<3.0.7": "3.0.7", - "shell-quote@<1.8.4": "1.8.4" + "shell-quote@<1.8.4": "1.8.4", + "esbuild@>=0.17.0 <0.28.1": "0.28.1" } }, "lint-staged": { diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index 912f978279..c16588556b 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -31,6 +31,7 @@ overrides: uuid@<11.1.1: 11.1.1 js-cookie@<3.0.7: 3.0.7 shell-quote@<1.8.4: 1.8.4 + esbuild@>=0.17.0 <0.28.1: 0.28.1 importers: @@ -148,8 +149,8 @@ importers: specifier: 1.0.0-rc.3 version: 1.0.0-rc.3(@opentelemetry/api@1.9.0)(@sinclair/typebox@0.34.41)(@types/pg@8.15.5)(pg@8.16.3(pg-native@3.5.2))(zod@4.1.12) esbuild: - specifier: ^0.27.2 - version: 0.27.2 + specifier: ^0.28.1 + version: 0.28.1 escape-html: specifier: ^1.0.3 version: 1.0.3 @@ -351,7 +352,7 @@ importers: version: 6.3.4 ts-jest: specifier: ^29.4.5 - version: 29.4.5(@babel/core@7.28.5)(@jest/transform@30.2.0)(@jest/types@30.2.0)(babel-jest@30.2.0(@babel/core@7.28.5))(esbuild@0.27.2)(jest-util@30.2.0)(jest@30.2.0(@types/node@22.19.1)(ts-node@10.9.2(@types/node@22.19.1)(typescript@5.9.2)))(typescript@5.9.2) + version: 29.4.5(@babel/core@7.28.5)(@jest/transform@30.2.0)(@jest/types@30.2.0)(babel-jest@30.2.0(@babel/core@7.28.5))(esbuild@0.28.1)(jest-util@30.2.0)(jest@30.2.0(@types/node@22.19.1)(ts-node@10.9.2(@types/node@22.19.1)(typescript@5.9.2)))(typescript@5.9.2) tsc-watch: specifier: ^7.1.1 version: 7.1.1(typescript@5.9.2) @@ -816,314 +817,158 @@ packages: '@emnapi/wasi-threads@1.1.0': resolution: {integrity: sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==} - '@esbuild/aix-ppc64@0.25.8': - resolution: {integrity: sha512-urAvrUedIqEiFR3FYSLTWQgLu5tb+m0qZw0NBEasUeo6wuqatkMDaRT+1uABiGXEu5vqgPd7FGE1BhsAIy9QVA==} + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} engines: {node: '>=18'} cpu: [ppc64] os: [aix] - '@esbuild/aix-ppc64@0.27.2': - resolution: {integrity: sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [aix] - - '@esbuild/android-arm64@0.25.8': - resolution: {integrity: sha512-OD3p7LYzWpLhZEyATcTSJ67qB5D+20vbtr6vHlHWSQYhKtzUYrETuWThmzFpZtFsBIxRvhO07+UgVA9m0i/O1w==} - engines: {node: '>=18'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm64@0.27.2': - resolution: {integrity: sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==} + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} engines: {node: '>=18'} cpu: [arm64] os: [android] - '@esbuild/android-arm@0.25.8': - resolution: {integrity: sha512-RONsAvGCz5oWyePVnLdZY/HHwA++nxYWIX1atInlaW6SEkwq6XkP3+cb825EUcRs5Vss/lGh/2YxAb5xqc07Uw==} - engines: {node: '>=18'} - cpu: [arm] - os: [android] - - '@esbuild/android-arm@0.27.2': - resolution: {integrity: sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==} + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} engines: {node: '>=18'} cpu: [arm] os: [android] - '@esbuild/android-x64@0.25.8': - resolution: {integrity: sha512-yJAVPklM5+4+9dTeKwHOaA+LQkmrKFX96BM0A/2zQrbS6ENCmxc4OVoBs5dPkCCak2roAD+jKCdnmOqKszPkjA==} - engines: {node: '>=18'} - cpu: [x64] - os: [android] - - '@esbuild/android-x64@0.27.2': - resolution: {integrity: sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==} + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} engines: {node: '>=18'} cpu: [x64] os: [android] - '@esbuild/darwin-arm64@0.25.8': - resolution: {integrity: sha512-Jw0mxgIaYX6R8ODrdkLLPwBqHTtYHJSmzzd+QeytSugzQ0Vg4c5rDky5VgkoowbZQahCbsv1rT1KW72MPIkevw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-arm64@0.27.2': - resolution: {integrity: sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==} + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] - '@esbuild/darwin-x64@0.25.8': - resolution: {integrity: sha512-Vh2gLxxHnuoQ+GjPNvDSDRpoBCUzY4Pu0kBqMBDlK4fuWbKgGtmDIeEC081xi26PPjn+1tct+Bh8FjyLlw1Zlg==} + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} engines: {node: '>=18'} cpu: [x64] os: [darwin] - '@esbuild/darwin-x64@0.27.2': - resolution: {integrity: sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==} - engines: {node: '>=18'} - cpu: [x64] - os: [darwin] - - '@esbuild/freebsd-arm64@0.25.8': - resolution: {integrity: sha512-YPJ7hDQ9DnNe5vxOm6jaie9QsTwcKedPvizTVlqWG9GBSq+BuyWEDazlGaDTC5NGU4QJd666V0yqCBL2oWKPfA==} - engines: {node: '>=18'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-arm64@0.27.2': - resolution: {integrity: sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==} + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] - '@esbuild/freebsd-x64@0.25.8': - resolution: {integrity: sha512-MmaEXxQRdXNFsRN/KcIimLnSJrk2r5H8v+WVafRWz5xdSVmWLoITZQXcgehI2ZE6gioE6HirAEToM/RvFBeuhw==} - engines: {node: '>=18'} - cpu: [x64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.27.2': - resolution: {integrity: sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==} + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] - '@esbuild/linux-arm64@0.25.8': - resolution: {integrity: sha512-WIgg00ARWv/uYLU7lsuDK00d/hHSfES5BzdWAdAig1ioV5kaFNrtK8EqGcUBJhYqotlUByUKz5Qo6u8tt7iD/w==} - engines: {node: '>=18'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm64@0.27.2': - resolution: {integrity: sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==} + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} engines: {node: '>=18'} cpu: [arm64] os: [linux] - '@esbuild/linux-arm@0.25.8': - resolution: {integrity: sha512-FuzEP9BixzZohl1kLf76KEVOsxtIBFwCaLupVuk4eFVnOZfU+Wsn+x5Ryam7nILV2pkq2TqQM9EZPsOBuMC+kg==} - engines: {node: '>=18'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-arm@0.27.2': - resolution: {integrity: sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==} + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} engines: {node: '>=18'} cpu: [arm] os: [linux] - '@esbuild/linux-ia32@0.25.8': - resolution: {integrity: sha512-A1D9YzRX1i+1AJZuFFUMP1E9fMaYY+GnSQil9Tlw05utlE86EKTUA7RjwHDkEitmLYiFsRd9HwKBPEftNdBfjg==} + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} engines: {node: '>=18'} cpu: [ia32] os: [linux] - '@esbuild/linux-ia32@0.27.2': - resolution: {integrity: sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==} - engines: {node: '>=18'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-loong64@0.25.8': - resolution: {integrity: sha512-O7k1J/dwHkY1RMVvglFHl1HzutGEFFZ3kNiDMSOyUrB7WcoHGf96Sh+64nTRT26l3GMbCW01Ekh/ThKM5iI7hQ==} - engines: {node: '>=18'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-loong64@0.27.2': - resolution: {integrity: sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==} + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} engines: {node: '>=18'} cpu: [loong64] os: [linux] - '@esbuild/linux-mips64el@0.25.8': - resolution: {integrity: sha512-uv+dqfRazte3BzfMp8PAQXmdGHQt2oC/y2ovwpTteqrMx2lwaksiFZ/bdkXJC19ttTvNXBuWH53zy/aTj1FgGw==} - engines: {node: '>=18'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-mips64el@0.27.2': - resolution: {integrity: sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==} + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] - '@esbuild/linux-ppc64@0.25.8': - resolution: {integrity: sha512-GyG0KcMi1GBavP5JgAkkstMGyMholMDybAf8wF5A70CALlDM2p/f7YFE7H92eDeH/VBtFJA5MT4nRPDGg4JuzQ==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-ppc64@0.27.2': - resolution: {integrity: sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==} + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] - '@esbuild/linux-riscv64@0.25.8': - resolution: {integrity: sha512-rAqDYFv3yzMrq7GIcen3XP7TUEG/4LK86LUPMIz6RT8A6pRIDn0sDcvjudVZBiiTcZCY9y2SgYX2lgK3AF+1eg==} - engines: {node: '>=18'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-riscv64@0.27.2': - resolution: {integrity: sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==} + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] - '@esbuild/linux-s390x@0.25.8': - resolution: {integrity: sha512-Xutvh6VjlbcHpsIIbwY8GVRbwoviWT19tFhgdA7DlenLGC/mbc3lBoVb7jxj9Z+eyGqvcnSyIltYUrkKzWqSvg==} + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} engines: {node: '>=18'} cpu: [s390x] os: [linux] - '@esbuild/linux-s390x@0.27.2': - resolution: {integrity: sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==} - engines: {node: '>=18'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-x64@0.25.8': - resolution: {integrity: sha512-ASFQhgY4ElXh3nDcOMTkQero4b1lgubskNlhIfJrsH5OKZXDpUAKBlNS0Kx81jwOBp+HCeZqmoJuihTv57/jvQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [linux] - - '@esbuild/linux-x64@0.27.2': - resolution: {integrity: sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==} + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} engines: {node: '>=18'} cpu: [x64] os: [linux] - '@esbuild/netbsd-arm64@0.25.8': - resolution: {integrity: sha512-d1KfruIeohqAi6SA+gENMuObDbEjn22olAR7egqnkCD9DGBG0wsEARotkLgXDu6c4ncgWTZJtN5vcgxzWRMzcw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [netbsd] - - '@esbuild/netbsd-arm64@0.27.2': - resolution: {integrity: sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==} + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] - '@esbuild/netbsd-x64@0.25.8': - resolution: {integrity: sha512-nVDCkrvx2ua+XQNyfrujIG38+YGyuy2Ru9kKVNyh5jAys6n+l44tTtToqHjino2My8VAY6Lw9H7RI73XFi66Cg==} - engines: {node: '>=18'} - cpu: [x64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.27.2': - resolution: {integrity: sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==} + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} engines: {node: '>=18'} cpu: [x64] os: [netbsd] - '@esbuild/openbsd-arm64@0.25.8': - resolution: {integrity: sha512-j8HgrDuSJFAujkivSMSfPQSAa5Fxbvk4rgNAS5i3K+r8s1X0p1uOO2Hl2xNsGFppOeHOLAVgYwDVlmxhq5h+SQ==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openbsd] - - '@esbuild/openbsd-arm64@0.27.2': - resolution: {integrity: sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==} + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] - '@esbuild/openbsd-x64@0.25.8': - resolution: {integrity: sha512-1h8MUAwa0VhNCDp6Af0HToI2TJFAn1uqT9Al6DJVzdIBAd21m/G0Yfc77KDM3uF3T/YaOgQq3qTJHPbTOInaIQ==} + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} engines: {node: '>=18'} cpu: [x64] os: [openbsd] - '@esbuild/openbsd-x64@0.27.2': - resolution: {integrity: sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==} - engines: {node: '>=18'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openharmony-arm64@0.25.8': - resolution: {integrity: sha512-r2nVa5SIK9tSWd0kJd9HCffnDHKchTGikb//9c7HX+r+wHYCpQrSgxhlY6KWV1nFo1l4KFbsMlHk+L6fekLsUg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openharmony] - - '@esbuild/openharmony-arm64@0.27.2': - resolution: {integrity: sha512-LRBbCmiU51IXfeXk59csuX/aSaToeG7w48nMwA6049Y4J4+VbWALAuXcs+qcD04rHDuSCSRKdmY63sruDS5qag==} + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] - '@esbuild/sunos-x64@0.25.8': - resolution: {integrity: sha512-zUlaP2S12YhQ2UzUfcCuMDHQFJyKABkAjvO5YSndMiIkMimPmxA+BYSBikWgsRpvyxuRnow4nS5NPnf9fpv41w==} - engines: {node: '>=18'} - cpu: [x64] - os: [sunos] - - '@esbuild/sunos-x64@0.27.2': - resolution: {integrity: sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==} + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} engines: {node: '>=18'} cpu: [x64] os: [sunos] - '@esbuild/win32-arm64@0.25.8': - resolution: {integrity: sha512-YEGFFWESlPva8hGL+zvj2z/SaK+pH0SwOM0Nc/d+rVnW7GSTFlLBGzZkuSU9kFIGIo8q9X3ucpZhu8PDN5A2sQ==} - engines: {node: '>=18'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-arm64@0.27.2': - resolution: {integrity: sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==} + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} engines: {node: '>=18'} cpu: [arm64] os: [win32] - '@esbuild/win32-ia32@0.25.8': - resolution: {integrity: sha512-hiGgGC6KZ5LZz58OL/+qVVoZiuZlUYlYHNAmczOm7bs2oE1XriPFi5ZHHrS8ACpV5EjySrnoCKmcbQMN+ojnHg==} - engines: {node: '>=18'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-ia32@0.27.2': - resolution: {integrity: sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==} + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} engines: {node: '>=18'} cpu: [ia32] os: [win32] - '@esbuild/win32-x64@0.25.8': - resolution: {integrity: sha512-cn3Yr7+OaaZq1c+2pe+8yxC8E144SReCQjN6/2ynubzYjvyqZjTXfQJpAcQpsdJq3My7XADANiYGHoFC69pLQw==} - engines: {node: '>=18'} - cpu: [x64] - os: [win32] - - '@esbuild/win32-x64@0.27.2': - resolution: {integrity: sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==} + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} engines: {node: '>=18'} cpu: [x64] os: [win32] @@ -3818,13 +3663,8 @@ packages: es6-promisify@5.0.0: resolution: {integrity: sha512-C+d6UdsYDk0lMebHNR4S2NybQMMngAOnOwYBQjTOiv0MkoJMP0Myw2mgpDLBcpfCmRLxyFqYhS/CfOENq4SJhQ==} - esbuild@0.25.8: - resolution: {integrity: sha512-vVC0USHGtMi8+R4Kz8rt6JhEWLxsv9Rnu/lGYbPR8u47B+DCBksq9JarW0zOO7bs37hyOK1l2/oqtbciutL5+Q==} - engines: {node: '>=18'} - hasBin: true - - esbuild@0.27.2: - resolution: {integrity: sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==} + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} engines: {node: '>=18'} hasBin: true @@ -6537,160 +6377,82 @@ snapshots: tslib: 2.8.1 optional: true - '@esbuild/aix-ppc64@0.25.8': - optional: true - - '@esbuild/aix-ppc64@0.27.2': - optional: true - - '@esbuild/android-arm64@0.25.8': - optional: true - - '@esbuild/android-arm64@0.27.2': - optional: true - - '@esbuild/android-arm@0.25.8': - optional: true - - '@esbuild/android-arm@0.27.2': - optional: true - - '@esbuild/android-x64@0.25.8': - optional: true - - '@esbuild/android-x64@0.27.2': - optional: true - - '@esbuild/darwin-arm64@0.25.8': - optional: true - - '@esbuild/darwin-arm64@0.27.2': - optional: true - - '@esbuild/darwin-x64@0.25.8': - optional: true - - '@esbuild/darwin-x64@0.27.2': - optional: true - - '@esbuild/freebsd-arm64@0.25.8': - optional: true - - '@esbuild/freebsd-arm64@0.27.2': - optional: true - - '@esbuild/freebsd-x64@0.25.8': - optional: true - - '@esbuild/freebsd-x64@0.27.2': - optional: true - - '@esbuild/linux-arm64@0.25.8': - optional: true - - '@esbuild/linux-arm64@0.27.2': - optional: true - - '@esbuild/linux-arm@0.25.8': - optional: true - - '@esbuild/linux-arm@0.27.2': - optional: true - - '@esbuild/linux-ia32@0.25.8': - optional: true - - '@esbuild/linux-ia32@0.27.2': - optional: true - - '@esbuild/linux-loong64@0.25.8': - optional: true - - '@esbuild/linux-loong64@0.27.2': - optional: true - - '@esbuild/linux-mips64el@0.25.8': + '@esbuild/aix-ppc64@0.28.1': optional: true - '@esbuild/linux-mips64el@0.27.2': + '@esbuild/android-arm64@0.28.1': optional: true - '@esbuild/linux-ppc64@0.25.8': + '@esbuild/android-arm@0.28.1': optional: true - '@esbuild/linux-ppc64@0.27.2': + '@esbuild/android-x64@0.28.1': optional: true - '@esbuild/linux-riscv64@0.25.8': + '@esbuild/darwin-arm64@0.28.1': optional: true - '@esbuild/linux-riscv64@0.27.2': + '@esbuild/darwin-x64@0.28.1': optional: true - '@esbuild/linux-s390x@0.25.8': + '@esbuild/freebsd-arm64@0.28.1': optional: true - '@esbuild/linux-s390x@0.27.2': + '@esbuild/freebsd-x64@0.28.1': optional: true - '@esbuild/linux-x64@0.25.8': + '@esbuild/linux-arm64@0.28.1': optional: true - '@esbuild/linux-x64@0.27.2': + '@esbuild/linux-arm@0.28.1': optional: true - '@esbuild/netbsd-arm64@0.25.8': + '@esbuild/linux-ia32@0.28.1': optional: true - '@esbuild/netbsd-arm64@0.27.2': + '@esbuild/linux-loong64@0.28.1': optional: true - '@esbuild/netbsd-x64@0.25.8': + '@esbuild/linux-mips64el@0.28.1': optional: true - '@esbuild/netbsd-x64@0.27.2': + '@esbuild/linux-ppc64@0.28.1': optional: true - '@esbuild/openbsd-arm64@0.25.8': + '@esbuild/linux-riscv64@0.28.1': optional: true - '@esbuild/openbsd-arm64@0.27.2': + '@esbuild/linux-s390x@0.28.1': optional: true - '@esbuild/openbsd-x64@0.25.8': + '@esbuild/linux-x64@0.28.1': optional: true - '@esbuild/openbsd-x64@0.27.2': + '@esbuild/netbsd-arm64@0.28.1': optional: true - '@esbuild/openharmony-arm64@0.25.8': + '@esbuild/netbsd-x64@0.28.1': optional: true - '@esbuild/openharmony-arm64@0.27.2': + '@esbuild/openbsd-arm64@0.28.1': optional: true - '@esbuild/sunos-x64@0.25.8': + '@esbuild/openbsd-x64@0.28.1': optional: true - '@esbuild/sunos-x64@0.27.2': + '@esbuild/openharmony-arm64@0.28.1': optional: true - '@esbuild/win32-arm64@0.25.8': + '@esbuild/sunos-x64@0.28.1': optional: true - '@esbuild/win32-arm64@0.27.2': + '@esbuild/win32-arm64@0.28.1': optional: true - '@esbuild/win32-ia32@0.25.8': + '@esbuild/win32-ia32@0.28.1': optional: true - '@esbuild/win32-ia32@0.27.2': - optional: true - - '@esbuild/win32-x64@0.25.8': - optional: true - - '@esbuild/win32-x64@0.27.2': + '@esbuild/win32-x64@0.28.1': optional: true '@exodus/bytes@1.15.1(@noble/hashes@1.8.0)': @@ -9456,63 +9218,34 @@ snapshots: dependencies: es6-promise: 4.2.8 - esbuild@0.25.8: - optionalDependencies: - '@esbuild/aix-ppc64': 0.25.8 - '@esbuild/android-arm': 0.25.8 - '@esbuild/android-arm64': 0.25.8 - '@esbuild/android-x64': 0.25.8 - '@esbuild/darwin-arm64': 0.25.8 - '@esbuild/darwin-x64': 0.25.8 - '@esbuild/freebsd-arm64': 0.25.8 - '@esbuild/freebsd-x64': 0.25.8 - '@esbuild/linux-arm': 0.25.8 - '@esbuild/linux-arm64': 0.25.8 - '@esbuild/linux-ia32': 0.25.8 - '@esbuild/linux-loong64': 0.25.8 - '@esbuild/linux-mips64el': 0.25.8 - '@esbuild/linux-ppc64': 0.25.8 - '@esbuild/linux-riscv64': 0.25.8 - '@esbuild/linux-s390x': 0.25.8 - '@esbuild/linux-x64': 0.25.8 - '@esbuild/netbsd-arm64': 0.25.8 - '@esbuild/netbsd-x64': 0.25.8 - '@esbuild/openbsd-arm64': 0.25.8 - '@esbuild/openbsd-x64': 0.25.8 - '@esbuild/openharmony-arm64': 0.25.8 - '@esbuild/sunos-x64': 0.25.8 - '@esbuild/win32-arm64': 0.25.8 - '@esbuild/win32-ia32': 0.25.8 - '@esbuild/win32-x64': 0.25.8 - - esbuild@0.27.2: + esbuild@0.28.1: optionalDependencies: - '@esbuild/aix-ppc64': 0.27.2 - '@esbuild/android-arm': 0.27.2 - '@esbuild/android-arm64': 0.27.2 - '@esbuild/android-x64': 0.27.2 - '@esbuild/darwin-arm64': 0.27.2 - '@esbuild/darwin-x64': 0.27.2 - '@esbuild/freebsd-arm64': 0.27.2 - '@esbuild/freebsd-x64': 0.27.2 - '@esbuild/linux-arm': 0.27.2 - '@esbuild/linux-arm64': 0.27.2 - '@esbuild/linux-ia32': 0.27.2 - '@esbuild/linux-loong64': 0.27.2 - '@esbuild/linux-mips64el': 0.27.2 - '@esbuild/linux-ppc64': 0.27.2 - '@esbuild/linux-riscv64': 0.27.2 - '@esbuild/linux-s390x': 0.27.2 - '@esbuild/linux-x64': 0.27.2 - '@esbuild/netbsd-arm64': 0.27.2 - '@esbuild/netbsd-x64': 0.27.2 - '@esbuild/openbsd-arm64': 0.27.2 - '@esbuild/openbsd-x64': 0.27.2 - '@esbuild/openharmony-arm64': 0.27.2 - '@esbuild/sunos-x64': 0.27.2 - '@esbuild/win32-arm64': 0.27.2 - '@esbuild/win32-ia32': 0.27.2 - '@esbuild/win32-x64': 0.27.2 + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 escalade@3.2.0: {} @@ -11703,7 +11436,7 @@ snapshots: triple-beam@1.4.1: {} - ts-jest@29.4.5(@babel/core@7.28.5)(@jest/transform@30.2.0)(@jest/types@30.2.0)(babel-jest@30.2.0(@babel/core@7.28.5))(esbuild@0.27.2)(jest-util@30.2.0)(jest@30.2.0(@types/node@22.19.1)(ts-node@10.9.2(@types/node@22.19.1)(typescript@5.9.2)))(typescript@5.9.2): + ts-jest@29.4.5(@babel/core@7.28.5)(@jest/transform@30.2.0)(@jest/types@30.2.0)(babel-jest@30.2.0(@babel/core@7.28.5))(esbuild@0.28.1)(jest-util@30.2.0)(jest@30.2.0(@types/node@22.19.1)(ts-node@10.9.2(@types/node@22.19.1)(typescript@5.9.2)))(typescript@5.9.2): dependencies: bs-logger: 0.2.6 fast-json-stable-stringify: 2.1.0 @@ -11721,7 +11454,7 @@ snapshots: '@jest/transform': 30.2.0 '@jest/types': 30.2.0 babel-jest: 30.2.0(@babel/core@7.28.5) - esbuild: 0.27.2 + esbuild: 0.28.1 jest-util: 30.2.0 ts-node@10.9.2(@types/node@22.19.1)(typescript@5.9.2): @@ -11759,7 +11492,7 @@ snapshots: tsx@4.20.3: dependencies: - esbuild: 0.25.8 + esbuild: 0.28.1 get-tsconfig: 4.10.1 optionalDependencies: fsevents: 2.3.3 diff --git a/apps/api/src/__tests__/snips/index-cache.test.ts b/apps/api/src/__tests__/snips/index-cache.test.ts index 12e0496e80..e1bb881eb9 100644 --- a/apps/api/src/__tests__/snips/index-cache.test.ts +++ b/apps/api/src/__tests__/snips/index-cache.test.ts @@ -4,6 +4,9 @@ import { filterIndexEntries, getCachedIndexEntries, getCachedMaxAge, + getCachedNegative, + isNegativeStillValid, + setCachedNegative, upsertCachedIndexEntries, type IndexCacheEntry, } from "../../services/index-cache"; @@ -187,6 +190,36 @@ describe("filterIndexEntries", () => { }); }); +describe("isNegativeStillValid", () => { + // A marker written at Tq with maxAge Mq stores emptyFrom = Tq - Mq, the left + // edge of the confirmed-empty window. A later request is still-empty iff its + // own left edge (now - maxAge) is no earlier than emptyFrom. + const Tq = 1_000_000_000_000; + const Mq = 2 * 24 * 60 * 60 * 1000; // 2 days + const emptyFrom = Tq - Mq; + + it("stays valid as time passes with the same maxAge window", () => { + // 1 hour later, same 2-day window: left edge moved forward, still >= emptyFrom. + expect(isNegativeStillValid(emptyFrom, Mq, Tq + 60 * 60 * 1000)).toBe(true); + }); + + it("stays valid for a smaller (more recent) window", () => { + expect(isNegativeStillValid(emptyFrom, 60 * 60 * 1000, Tq)).toBe(true); + }); + + it("is invalid for a larger window that looks further back than confirmed", () => { + // 14-day window now reaches earlier than the 2-day confirmed-empty left edge. + expect( + isNegativeStillValid(emptyFrom, 14 * 24 * 60 * 60 * 1000, Tq), + ).toBe(false); + }); + + it("is exactly valid at the boundary", () => { + // now - maxAge == emptyFrom + expect(isNegativeStillValid(emptyFrom, Mq, Tq)).toBe(true); + }); +}); + describe("index cache fail-open", () => { // Points at a port nothing listens on: every operation must resolve (not // throw, not hang) so the read path can fall back to Postgres. @@ -227,4 +260,16 @@ describe("index cache fail-open", () => { getCachedMaxAge(urlHash, undefined, deadClient), ).resolves.toBeNull(); }, 5000); + + it("getCachedNegative resolves to null (disabled or dead) and never throws", async () => { + await expect( + getCachedNegative("idxc:test", undefined, deadClient), + ).resolves.toBeNull(); + }, 5000); + + it("setCachedNegative resolves without throwing", async () => { + await expect( + setCachedNegative("idxc:test", Date.now(), undefined, deadClient), + ).resolves.toBeUndefined(); + }, 5000); }); diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index c3e1872c05..ac6c4bacce 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -85,6 +85,11 @@ const configSchema = z.object({ DATABASE_REPLICA_URL: z.string().optional(), INDEX_DATABASE_URL: z.string().optional(), INDEX_CACHE_REDIS_URL: z.string().optional(), + // Negative (miss) caching TTL for index URL->id lookups, in ms. 0 disables + // it; the cache then only shields lookups that find data. A positive value + // (e.g. 600000 = 10min) also short-circuits repeat lookups for URLs with no + // index entry. Kept short so any missed cache-clear self-heals quickly. + INDEX_CACHE_NEGATIVE_TTL_MS: z.coerce.number().default(0), REDIS_URL: z.string().optional(), REDIS_EVICT_URL: z.string().optional(), REDIS_RATE_LIMIT_URL: z.string().optional(), @@ -103,6 +108,9 @@ const configSchema = z.object({ CLICKHOUSE_ANALYTICS_URL: z.string().optional(), CLICKHOUSE_ANALYTICS_DATABASE: z.string().optional(), + // Search highlights (beta): semantic highlight model endpoint + HIGHLIGHT_MODEL_URL: z.string().optional(), + // Fire Engine FIRE_ENGINE_BETA_URL: z.string().optional(), FIRE_ENGINE_STAGING_URL: z.string().optional(), diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 117c69c763..ff1de3ac87 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -1310,6 +1310,7 @@ export type TeamFlags = { // POST /v2/search/:jobId/feedback returns 403 TEAM_OPTED_OUT when true. searchFeedbackOptOut?: boolean; researchBeta?: boolean; + highlightsBeta?: boolean; } | null; export type AuthCreditUsageChunkFromTeam = Omit< diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts index 126fd6f318..9a79a57d86 100644 --- a/apps/api/src/controllers/v2/search.ts +++ b/apps/api/src/controllers/v2/search.ts @@ -135,6 +135,7 @@ export async function searchController( excludeDomains: req.body.excludeDomains, enterprise: req.body.enterprise, scrapeOptions: req.body.scrapeOptions, + highlights: req.body.highlights, timeout: req.body.timeout, }, { diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts index 82045bc6ae..764d7020a3 100644 --- a/apps/api/src/controllers/v2/types.ts +++ b/apps/api/src/controllers/v2/types.ts @@ -1525,6 +1525,7 @@ export type TeamFlags = { debugBranding?: boolean; maxBrowserSessions?: number; researchBeta?: boolean; + highlightsBeta?: boolean; } | null; interface RequestWithMaybeACUC< @@ -1905,6 +1906,10 @@ export const searchRequestSchema = z timeout: z.int().positive().finite().prefault(60000), ignoreInvalidURLs: z.boolean().optional().prefault(false), asyncScraping: z.boolean().optional().prefault(false), + // Experimental: replace each result's snippet with query-relevant + // highlights pulled from our index (last 30 days), out-of-line from + // scrapeURL. Falls back to the provider snippet when the URL isn't indexed. + highlights: z.boolean().optional().prefault(false), __searchPreviewToken: z.string().optional(), scrapeOptions: baseScrapeOptions .extend({ diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts index 9f4c72cb24..70361b069f 100644 --- a/apps/api/src/lib/scrape-billing.test.ts +++ b/apps/api/src/lib/scrape-billing.test.ts @@ -24,4 +24,66 @@ describe("calculateCreditsToBeBilled", () => { expect(credits).toBe(30); }); + + it("bills deterministic JSON at 10 credits when the script was generated", async () => { + const credits = await calculateCreditsToBeBilled( + { + formats: [{ type: "deterministicJson", schema: {} }], + } as any, + { + teamId: "team-id", + }, + { + metadata: { + statusCode: 200, + proxyUsed: "basic", + }, + } as any, + { + totalCost: 0.01, + calls: [ + { + type: "other", + model: "vertex/gemini", + cost: 0.01, + metadata: { module: "deterministic-json", role: "codegen" }, + }, + ], + } as any, + {} as any, + ); + + expect(credits).toBe(10); + }); + + it("bills deterministic JSON at 3 credits when a cached script was reused", async () => { + const credits = await calculateCreditsToBeBilled( + { + formats: [{ type: "deterministicJson", schema: {} }], + } as any, + { + teamId: "team-id", + }, + { + metadata: { + statusCode: 200, + proxyUsed: "basic", + }, + } as any, + { + totalCost: 0.001, + calls: [ + { + type: "other", + model: "groq/llama", + cost: 0.001, + metadata: { module: "deterministic-json", role: "askLlm" }, + }, + ], + } as any, + {} as any, + ); + + expect(credits).toBe(3); + }); }); diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts index 3efa050458..2de00540a2 100644 --- a/apps/api/src/lib/scrape-billing.ts +++ b/apps/api/src/lib/scrape-billing.ts @@ -73,7 +73,14 @@ export async function calculateCreditsToBeBilled( } if (hasFormatOfType(options.formats, "deterministicJson")) { - creditsToBeBilled = 7; + // 10 when this run generated the extractor script, 3 when it reused a + // cached one. The codegen call is tagged in deterministicJson/llm/client.ts. + const generatedScript = costTrackingJSON.calls?.some( + call => + call.metadata?.module === "deterministic-json" && + call.metadata?.role === "codegen", + ); + creditsToBeBilled = generatedScript ? 10 : 3; } if ( diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index eb499ed67b..0f607d1f57 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -20,9 +20,13 @@ import { filterIndexEntries, getCachedIndexEntries, getCachedMaxAge, + getCachedNegative, + isNegativeStillValid, setCachedMaxAge, + setCachedNegative, upsertCachedIndexEntries, useIndexCache, + useIndexNegativeCache, type IndexCacheEntry, } from "../../../../services/index-cache"; import { indexLookupCounter } from "../../../../lib/index-cache-metrics"; @@ -314,6 +318,7 @@ export async function scrapeURLWithIndex( let cacheStatus: "hit" | "filtered" | "miss" | "error" | "disabled" = "disabled"; let servedFromCache = false; + let negativeHit = false; let timingsCache = 0; let timingsDb = 0; @@ -325,8 +330,9 @@ export async function scrapeURLWithIndex( dbResult: "hit" | "miss" | "error", extra: Record = {}, ) => { - const outcome = - cacheStatus === "hit" + const outcome = negativeHit + ? "cache_neg_hit" + : cacheStatus === "hit" ? "cache_hit" : cacheStatus === "disabled" ? `db_only_${dbResult}` @@ -384,7 +390,26 @@ export async function scrapeURLWithIndex( } } - if (!servedFromCache) { + // Negative cache: on a clean positive miss (key absent), a still-valid + // negative marker proves there's no index entry for this window, so we can + // skip Postgres. Not consulted on "filtered"/"error" (entries exist, or the + // cache is unhealthy and we must fall back to the DB), nor for minAge + // requests (different no-data semantics — NoCachedDataError, no waterfall). + if ( + !servedFromCache && + useIndexNegativeCache && + cacheStatus === "miss" && + meta.options.minAge === undefined + ) { + const negStart = Date.now(); + const neg = await getCachedNegative(variantKey, meta.logger); + timingsCache += Date.now() - negStart; + if (neg !== null && isNegativeStillValid(neg.emptyFrom, maxAge, Date.now())) { + negativeHit = true; + } + } + + if (!servedFromCache && !negativeHit) { const dbStart = Date.now(); try { const rows = await indexGetRecent5({ @@ -418,6 +443,15 @@ export async function scrapeURLWithIndex( upsertCachedIndexEntries(variantKey, entries, meta.logger).catch( () => {}, ); + } else if ( + useIndexNegativeCache && + rows.length === 0 && + meta.options.minAge === undefined + ) { + // Confirmed empty for [dbStart - maxAge, dbStart]; record the left edge. + setCachedNegative(variantKey, dbStart - maxAge, meta.logger).catch( + () => {}, + ); } data = rows; } catch (error) { diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts index 789b4272de..c252c467bf 100644 --- a/apps/api/src/search/execute.ts +++ b/apps/api/src/search/execute.ts @@ -13,6 +13,7 @@ import { mergeScrapedContent, calculateScrapeCredits, } from "./scrape"; +import { applySearchHighlights, highlightsEnvReady } from "./highlights"; import { trackSearchResults, trackSearchRequest } from "../lib/tracking"; import type { BillingMetadata } from "../services/billing/types"; @@ -30,6 +31,7 @@ interface SearchOptions { excludeDomains?: string[]; enterprise?: ("default" | "anon" | "zdr")[]; scrapeOptions?: ScrapeOptions; + highlights?: boolean; timeout: number; } @@ -185,6 +187,17 @@ export async function executeSearch( } } + // Experimental highlights beta: replace provider snippets with index-backed + // highlights. Gated on (1) the request opting in, (2) the team's highlightsBeta + // flag, and (3) all required envs being present (index DB, GCS, model). Any + // gate failing => silently keep the provider snippets. + // Runs after scraping (mergeScrapedContent rebuilds the result objects, so + // highlight mutations must come last to survive). Uses the user's original + // query, not the domain-filtered upstream query. + if (options.highlights && flags?.highlightsBeta === true && highlightsEnvReady()) { + await applySearchHighlights(searchResponse, query, logger); + } + const scrapeFormats = scrapeOptions?.formats ? scrapeOptions.formats.map((f: any) => typeof f === "string" ? f : f.type, diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.ts new file mode 100644 index 0000000000..817c37f038 --- /dev/null +++ b/apps/api/src/search/highlight-model.ts @@ -0,0 +1,203 @@ +import type { Logger } from "winston"; +import { config } from "../config"; + +// Semantic highlight model: it scores each sentence of a context against a +// question (semantic similarity), no LLM. Inference is fast (~10ms), but the +// model caps context at ~4k tokens, so we chunk the page, score every chunk, +// pool the per-sentence scores globally, and keep the best sentences. Endpoint +// is config-gated (HIGHLIGHT_MODEL_URL); callers must confirm it's set via +// highlightsEnvReady() before invoking. + +// ~4 chars/token; 10k chars (~2.5k tokens) leaves headroom under the 4k cap for +// the question + tokenizer variance (markdown/code tokenizes denser than prose). +const CHUNK_CHARS = 10000; +// Bound load for very long pages (chunks run concurrently per result). +const MAX_CHUNKS = 10; +// Keep sentences scoring at/above this (matches the model's default threshold). +const SELECT_THRESHOLD = 0.5; +// Cap the assembled snippet length. +const MAX_SELECTED = 12; +const REQUEST_TIMEOUT_MS = 30000; + +interface ScoredSentence { + text: string; + score: number; + order: number; // global document order +} + +// Strip markdown syntax so a highlight reads as plain prose: links/images keep +// their text, bare URLs and emphasis/heading/code marks go, line-number gutters +// from rendered code blocks are dropped, whitespace collapses. +function cleanSentence(text: string): string { + return text + .replace(/!\[[^\]]*\]\([^)]*\)/g, " ") // images + .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") // links -> link text + .replace(/https?:\/\/\S+/g, " ") // bare URLs + .replace(/^\s*\d+\s*/, "") // leading code-block line number + .replace(/[#*`>_~|\\]+/g, " ") // markdown marks + .replace(/\s+/g, " ") + .trim(); +} + +// Drop markdown/code artifacts (line numbers, "=====", "//", "[ 01 / 06 ]") +// that the semantic model sometimes scores highly. Real sentences have prose. +function isContentful(text: string): boolean { + const letters = (text.match(/[a-zA-Z]/g) ?? []).length; + return letters >= 15; +} + +interface HighlightModelResponse { + kept_sentences?: string[]; + sentence_probabilities?: number[]; +} + +/** + * Split markdown into chunks under the model's token budget. Splits on blank + * lines (paragraph boundaries) to avoid cutting mid-sentence; oversized + * paragraphs are hard-split as a last resort. Capped at MAX_CHUNKS. + */ +function chunkMarkdown(markdown: string): string[] { + const chunks: string[] = []; + let current = ""; + + const flush = () => { + if (current.trim() !== "") chunks.push(current); + current = ""; + }; + + for (const para of markdown.split(/\n{2,}/)) { + if (chunks.length >= MAX_CHUNKS) break; + + if (para.length > CHUNK_CHARS) { + flush(); + for ( + let i = 0; + i < para.length && chunks.length < MAX_CHUNKS; + i += CHUNK_CHARS + ) { + chunks.push(para.slice(i, i + CHUNK_CHARS)); + } + continue; + } + + if (current.length + para.length + 2 > CHUNK_CHARS) { + flush(); + } + current += (current === "" ? "" : "\n\n") + para; + } + + if (chunks.length < MAX_CHUNKS) flush(); + return chunks.slice(0, MAX_CHUNKS); +} + +async function scoreChunk( + question: string, + context: string, + logger: Logger, +): Promise<{ text: string; score: number }[]> { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); + try { + const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/v1/highlight`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + question, + context, + // threshold 0 => every sentence is returned with its score, so we can + // pool scores across chunks and select globally. + threshold: 0.0, + language: "auto", + return_sentence_metrics: true, + }), + signal: controller.signal, + }); + + if (!res.ok) { + const body = await res.text().catch(() => ""); + throw new Error( + `highlight model HTTP ${res.status}: ${body.slice(0, 200)}`, + ); + } + + const data = (await res.json()) as HighlightModelResponse; + const sentences = data.kept_sentences ?? []; + const probs = data.sentence_probabilities ?? []; + const n = Math.min(sentences.length, probs.length); + const out: { text: string; score: number }[] = []; + for (let i = 0; i < n; i++) { + out.push({ text: sentences[i], score: probs[i] }); + } + return out; + } finally { + clearTimeout(timer); + } +} + +/** + * Generate query-relevant highlights from page markdown using the semantic + * highlight model. Returns the assembled highlight string (best sentences in + * document order), or null if nothing clears the threshold / all chunks fail. + */ +export async function generateSemanticHighlights( + markdown: string, + query: string, + opts: { logger: Logger }, +): Promise { + const chunks = chunkMarkdown(markdown); + if (chunks.length === 0) return null; + + const start = Date.now(); + const perChunk = await Promise.all( + chunks.map(async (chunk, idx) => { + try { + return await scoreChunk(query, chunk, opts.logger); + } catch (error) { + opts.logger.warn("highlight model chunk failed", { + error: error instanceof Error ? error.message : String(error), + chunkIdx: idx, + }); + return []; + } + }), + ); + + // Pool every scored sentence, preserving document order (Promise.all keeps + // chunk order, and sentences within a chunk are already in order). + const all: ScoredSentence[] = []; + let order = 0; + for (const chunkSentences of perChunk) { + for (const s of chunkSentences) { + const idx = order++; + const cleaned = cleanSentence(s.text); + if (isContentful(cleaned)) { + all.push({ text: cleaned, score: s.score, order: idx }); + } + } + } + if (all.length === 0) return null; + + // Pick the best: keep sentences at/above the threshold, cap to the top + // MAX_SELECTED by score, then restore document order for readability. + let selected = all.filter(s => s.score >= SELECT_THRESHOLD); + if (selected.length === 0) return null; + selected.sort((a, b) => b.score - a.score); + selected = selected.slice(0, MAX_SELECTED); + selected.sort((a, b) => a.order - b.order); + + const text = selected + .map(s => s.text.trim()) + .join(" ") + .replace(/\s+/g, " ") + .trim(); + + opts.logger.info("semantic highlights generated", { + chunks: chunks.length, + scoredSentences: all.length, + selected: selected.length, + topScore: all.reduce((m, s) => Math.max(m, s.score), 0), + elapsedMs: Date.now() - start, + }); + + return text === "" ? null : text; +} diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.ts new file mode 100644 index 0000000000..116f5ff0d0 --- /dev/null +++ b/apps/api/src/search/highlights.ts @@ -0,0 +1,187 @@ +import type { Logger } from "winston"; +import { SearchV2Response } from "../lib/entities"; +import { + normalizeURLForIndex, + hashURL, + getIndexFromGCS, + useIndex, +} from "../services"; +import { indexGetRecent5 } from "../db/rpc"; +import { parseMarkdown } from "../lib/html-to-markdown"; +import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements"; +import type { ScrapeOptions } from "../controllers/v2/types"; +import { generateSemanticHighlights } from "./highlight-model"; +import { config } from "../config"; + +// How far back into the index we're willing to reach for highlight source text. +const HIGHLIGHTS_INDEX_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days + +/** + * Whether the deployment has every dependency the highlights beta needs: the + * index DB (to find cached content), the GCS index bucket (to fetch it), and the + * highlight model endpoint (to score it). Missing any => silently skip. + */ +export function highlightsEnvReady(): boolean { + return ( + useIndex && !!config.GCS_INDEX_BUCKET_NAME && !!config.HIGHLIGHT_MODEL_URL + ); +} + +// Mirrors scrapeURLWithIndex: prefer the newest 2xx entry unless it sits behind +// this many more-recent error entries, in which case we surface the newest one. +const ERROR_COUNT_TO_REGISTER = 3; + +// This whole module runs out-of-line from scrapeURL on purpose: it reads +// already-indexed content directly from the index DB + GCS instead of routing +// through the scrape engine. That keeps highlight generation off the critical +// scrape path and lets us experiment with latency freely. + +/** + * Fetch the most recent indexed markdown for a URL within the last 30 days. + * Returns null when the URL isn't in the index (or the lookup fails) so callers + * can fall back to the provider snippet. + */ +async function getIndexedMarkdownForURL( + url: string, + logger: Logger, +): Promise { + if (!useIndex) { + return null; + } + + try { + const normalizedURL = normalizeURLForIndex(url); + const urlHash = hashURL(normalizedURL); + + // Match the most common index variant (default scrape options) to maximize + // hit rate: desktop, ads blocked, no screenshot, no location, no stealth. + const rows = await indexGetRecent5({ + url_hash: urlHash, + max_age_ms: HIGHLIGHTS_INDEX_MAX_AGE_MS, + is_mobile: false, + block_ads: true, + feature_screenshot: false, + feature_screenshot_fullscreen: false, + location_country: null, + location_languages: null, + wait_time_ms: 0, + is_stealth: false, + min_age_ms: null, + }); + + if (!rows || rows.length === 0) { + return null; + } + + const newest200Index = rows.findIndex( + x => x.status >= 200 && x.status < 300, + ); + const selected = + newest200Index >= ERROR_COUNT_TO_REGISTER || newest200Index === -1 + ? rows[0] + : rows[newest200Index]; + + const doc = await getIndexFromGCS( + selected.id + ".json", + logger.child({ module: "search/highlights", method: "getIndexFromGCS" }), + ); + if (!doc || !doc.html) { + return null; + } + + // Skip raw base64 PDFs — they aren't useful as highlight source text. + if (typeof doc.html === "string" && doc.html.startsWith("JVBERi")) { + return null; + } + + // The index stores rawHtml, so we must run the same cleaning the scrape + // pipeline does (strip