diff --git a/package-lock.json b/package-lock.json index 6f7ca745..27fe8268 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10597,6 +10597,31 @@ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "dev": true }, + "node_modules/groq-sdk": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/groq-sdk/-/groq-sdk-0.5.0.tgz", + "integrity": "sha512-RVmhW7qZ+XZoy5fIuSdx/LGQJONpL8MHgZEW7dFwTdgkzStub2XQx6OKv28CHogijdwH41J+Npj/z2jBPu3vmw==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" + } + }, + "node_modules/groq-sdk/node_modules/@types/node": { + "version": "18.19.45", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.45.tgz", + "integrity": "sha512-VZxPKNNhjKmaC1SUYowuXSRSMGyQGmQjvvA1xE4QZ0xce2kLtEhPDS+kqpCPBZYgqblCLQ2DAjSzmgCM5auvhA==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/handlebars": { "version": "4.7.8", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", @@ -16660,6 +16685,7 @@ "express": "^4.18.2", "file-type": "^18.5.0", "form-data": "^4.0.0", + "groq-sdk": "^0.5.0", "handlebars": "^4.7.8", "helmet": "^7.0.0", "hi-base32": "^0.5.1", diff --git a/src/backend/package.json b/src/backend/package.json index 2bd8e544..d55ced4e 100644 --- a/src/backend/package.json +++ b/src/backend/package.json @@ -36,6 +36,7 @@ "express": "^4.18.2", "file-type": "^18.5.0", "form-data": "^4.0.0", + "groq-sdk": "^0.5.0", "handlebars": "^4.7.8", "helmet": "^7.0.0", "hi-base32": "^0.5.1", diff --git a/src/backend/src/modules/puterai/GroqAIService.js b/src/backend/src/modules/puterai/GroqAIService.js new file mode 100644 index 00000000..3426d779 --- /dev/null +++ b/src/backend/src/modules/puterai/GroqAIService.js @@ -0,0 +1,69 @@ +const { PassThrough } = require("stream"); +const BaseService = require("../../services/BaseService"); +const { TypedValue } = require("../../services/drivers/meta/Runtime"); +const { nou } = require("../../util/langutil"); + +class GroqAIService extends BaseService { + static MODULES = { + Groq: require('groq-sdk'), + } + + async _init () { + const Groq = require('groq-sdk'); + this.client = new Groq({ + apiKey: this.config.apiKey, + }); + } + + static IMPLEMENTS = { + 'puter-chat-completion': { + async list () { + // They send: { "object": "list", data } + const funny_wrapper = await this.client.models.list(); + return funny_wrapper.data; + }, + async complete ({ messages, model, stream }) { + for ( let i = 0; i < messages.length; i++ ) { + const message = messages[i]; + if ( ! message.role ) message.role = 'user'; + } + + const completion = await this.client.chat.completions.create({ + messages, + model, + stream, + }); + + if ( stream ) { + const stream = new PassThrough(); + const retval = new TypedValue({ + $: 'stream', + content_type: 'application/x-ndjson', + chunked: true, + }, stream); + (async () => { + for await ( const chunk of completion ) { + if ( chunk.choices.length < 1 ) continue; + if ( chunk.choices[0].finish_reason ) { + stream.end(); + break; + } + if ( nou(chunk.choices[0].delta.content) ) continue; + const str = JSON.stringify({ + text: chunk.choices[0].delta.content + }); + stream.write(str + '\n'); + } + })(); + return retval; + } + + return completion.choices[0]; + } + } + }; +} + +module.exports = { + GroqAIService, +}; diff --git a/src/backend/src/modules/puterai/PuterAIModule.js b/src/backend/src/modules/puterai/PuterAIModule.js index 2e0d2a48..fb88b64e 100644 --- a/src/backend/src/modules/puterai/PuterAIModule.js +++ b/src/backend/src/modules/puterai/PuterAIModule.js @@ -43,6 +43,11 @@ class PuterAIModule extends AdvancedBase { const { MistralAIService } = require('./MistralAIService'); services.registerService('mistral', MistralAIService); } + + if ( !! config?.services?.['groq'] ) { + const { GroqAIService } = require('./GroqAIService'); + services.registerService('groq', GroqAIService); + } } } diff --git a/src/puter-js/src/modules/AI.js b/src/puter-js/src/modules/AI.js index 59388318..a04a79d9 100644 --- a/src/puter-js/src/modules/AI.js +++ b/src/puter-js/src/modules/AI.js @@ -229,6 +229,9 @@ class AI{ if ( options.model === 'mistral' ) { options.model = 'mistral-large-latest'; } + if ( options.model === 'groq' ) { + options.model = 'llama3-8b-8192'; + } // map model to the appropriate driver if (!options.model || options.model === 'gpt-4o' || options.model === 'gpt-4o-mini') { @@ -239,6 +242,21 @@ class AI{ driver = 'together-ai'; }else if(options.model === 'mistral-large-latest' || options.model === 'codestral-latest'){ driver = 'mistral'; + }else if([ + "distil-whisper-large-v3-en", + "gemma2-9b-it", + "gemma-7b-it", + "llama-3.1-70b-versatile", + "llama-3.1-8b-instant", + "llama3-70b-8192", + "llama3-8b-8192", + "llama3-groq-70b-8192-tool-use-preview", + "llama3-groq-8b-8192-tool-use-preview", + "llama-guard-3-8b", + "mixtral-8x7b-32768", + "whisper-large-v3" + ].includes(options.model)) { + driver = 'groq'; } // stream flag from settings