diff --git a/package-lock.json b/package-lock.json index 307958d..7cea720 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,8 +12,7 @@ "@fluentui/react-components": "^9.54.17", "@fluentui/react-icons": "^2.0.260", "@sctg/ai-sdk": "0.0.3", - "@sctg/sentencepiece-js": "^1.3.1", - "buffer": "^6.0.3", + "@sctg/sentencepiece-js": "^1.3.3", "core-js": "^3.38.1", "es6-promise": "^4.2.8", "react": "^18.3.1", @@ -4447,9 +4446,9 @@ } }, "node_modules/@jsonjoy.com/util": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@jsonjoy.com/util/-/util-1.3.0.tgz", - "integrity": "sha512-Cebt4Vk7k1xHy87kHY7KSPLT77A7Ev7IfOblyLZhtYEhrdQ6fX4EoLq3xOQ3O/DRMEh2ok5nyC180E+ABS8Wmw==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/util/-/util-1.5.0.tgz", + "integrity": "sha512-ojoNsrIuPI9g6o8UxhraZQSyF2ByJanAY4cTFbc8Mf2AXEF4aQRGY1dJxyJpuyav8r9FGflEt/Ff3u5Nt6YMPA==", "dev": true, "license": "Apache-2.0", "engines": { @@ -5427,12 +5426,13 @@ } }, "node_modules/@sctg/sentencepiece-js": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/@sctg/sentencepiece-js/-/sentencepiece-js-1.3.1.tgz", - "integrity": "sha512-j44CcZDUCWXAhXKJJCp7KlHYiVztHiJ2cUq9un03Z+svp2w3fj68Z6PSOpYwZLJINPCcDRToZoZrz867xWJy7g==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@sctg/sentencepiece-js/-/sentencepiece-js-1.3.3.tgz", + "integrity": "sha512-iPnzR2HGjdQQG2SpTyPH3wnnpgQ2aS14B3I2jHjomkoaMprM3Sn+WOQblHwByetnvn3n14y3IwjwogVUHA2cVA==", "license": "Apache-2.0", "dependencies": { - "app-root-path": "^3.1.0" + "app-root-path": "^3.1.0", + "buffer": "^6.0.3" } }, "node_modules/@sigstore/bundle": { @@ -9249,6 +9249,7 @@ "version": "8.57.1", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", + "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", "dependencies": { @@ -11278,9 +11279,9 @@ } }, "node_modules/http-proxy-middleware": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz", - "integrity": "sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz", + "integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==", "dev": true, "license": "MIT", "dependencies": { @@ -18586,9 +18587,9 @@ } }, "node_modules/synckit": { - "version": "0.9.1", - "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.1.tgz", - "integrity": "sha512-7gr8p9TQP6RAHusBOSLs46F4564ZrjV8xFmw5zCmgmhGUcw2hxsShhJ6CEiHQMgPDwAQ1fWHPM0ypc4RMAig4A==", + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.2.tgz", + "integrity": "sha512-vrozgXDQwYO72vHjUb/HnFbQx1exDjoKzqx23aXEg2a9VIg2TSFZ8FmeZpTjUCFMYw7mpX4BE2SFu8wI7asYsw==", "dev": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index db79a85..bb024fc 100644 --- a/package.json +++ b/package.json @@ -29,8 +29,7 @@ "@fluentui/react-components": "^9.54.17", "@fluentui/react-icons": "^2.0.260", "@sctg/ai-sdk": "0.0.3", - "@sctg/sentencepiece-js": "^1.3.1", - "buffer": "^6.0.3", + "@sctg/sentencepiece-js": "^1.3.3", "core-js": "^3.38.1", "es6-promise": "^4.2.8", "react": "^18.3.1", diff --git a/src/aipane/aipane.ts b/src/aipane/aipane.ts index 1184d03..d58727a 100644 --- a/src/aipane/aipane.ts +++ b/src/aipane/aipane.ts @@ -7,19 +7,19 @@ import { Groq } from "@sctg/ai-sdk"; import config from "../config.json"; import type { AIModel, AIPrompt, AIProvider } from "./AIPrompt"; -import { SentencePieceProcessor, cleanText, llama_3_1_tokeniser_b64 } from "@sctg/sentencepiece-js"; -import { Buffer } from "buffer"; - -// eslint-disable-next-line no-undef -globalThis.Buffer = Buffer; -const TOKEN_MARGIN = 20; +import { SentencePieceProcessor, cleanText, llama_3_1_tokeniser_b64 } from "@sctg/sentencepiece-js"; +const TOKEN_MARGIN = 20; // Safety margin for token count async function countTokens(text: string): Promise { + // Remove invalid characters and normalise whitespace let cleaned = cleanText(text); + // Create a new SentencePieceProcessor let spp = new SentencePieceProcessor(); + // Load the tokeniser model from a base64 string await spp.loadFromB64StringModel(llama_3_1_tokeniser_b64); + // Encode the cleaned text into token IDs let ids = spp.encodeIds(cleaned); - return ids.length; + return ids.length; // Return the number of tokens } async function groqRequest(