feat(puterai): add groq

package-lock.json

@@ -10597,6 +10597,31 @@
       "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
       "dev": true
     },
+    "node_modules/groq-sdk": {
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/groq-sdk/-/groq-sdk-0.5.0.tgz",
+      "integrity": "sha512-RVmhW7qZ+XZoy5fIuSdx/LGQJONpL8MHgZEW7dFwTdgkzStub2XQx6OKv28CHogijdwH41J+Npj/z2jBPu3vmw==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@types/node": "^18.11.18",
+        "@types/node-fetch": "^2.6.4",
+        "abort-controller": "^3.0.0",
+        "agentkeepalive": "^4.2.1",
+        "form-data-encoder": "1.7.2",
+        "formdata-node": "^4.3.2",
+        "node-fetch": "^2.6.7",
+        "web-streams-polyfill": "^3.2.1"
+      }
+    },
+    "node_modules/groq-sdk/node_modules/@types/node": {
+      "version": "18.19.45",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.45.tgz",
+      "integrity": "sha512-VZxPKNNhjKmaC1SUYowuXSRSMGyQGmQjvvA1xE4QZ0xce2kLtEhPDS+kqpCPBZYgqblCLQ2DAjSzmgCM5auvhA==",
+      "license": "MIT",
+      "dependencies": {
+        "undici-types": "~5.26.4"
+      }
+    },
     "node_modules/handlebars": {
       "version": "4.7.8",
       "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz",
@@ -16660,6 +16685,7 @@
         "express": "^4.18.2",
         "file-type": "^18.5.0",
         "form-data": "^4.0.0",
+        "groq-sdk": "^0.5.0",
         "handlebars": "^4.7.8",
         "helmet": "^7.0.0",
         "hi-base32": "^0.5.1",

src/backend/package.json

@@ -36,6 +36,7 @@
     "express": "^4.18.2",
     "file-type": "^18.5.0",
     "form-data": "^4.0.0",
+    "groq-sdk": "^0.5.0",
     "handlebars": "^4.7.8",
     "helmet": "^7.0.0",
     "hi-base32": "^0.5.1",

src/backend/src/modules/puterai/GroqAIService.js

@@ -0,0 +1,69 @@
+const { PassThrough } = require("stream");
+const BaseService = require("../../services/BaseService");
+const { TypedValue } = require("../../services/drivers/meta/Runtime");
+const { nou } = require("../../util/langutil");
+
+class GroqAIService extends BaseService {
+    static MODULES = {
+        Groq: require('groq-sdk'),
+    }
+
+    async _init () {
+        const Groq = require('groq-sdk');
+        this.client = new Groq({
+            apiKey: this.config.apiKey,
+        });
+    }
+    
+    static IMPLEMENTS = {
+        'puter-chat-completion': {
+            async list () {
+                // They send: { "object": "list", data }
+                const funny_wrapper = await this.client.models.list();
+                return funny_wrapper.data;
+            },
+            async complete ({ messages, model, stream }) {
+                for ( let i = 0; i < messages.length; i++ ) {
+                    const message = messages[i];
+                    if ( ! message.role ) message.role = 'user';
+                }
+
+                const completion = await this.client.chat.completions.create({
+                    messages,
+                    model,
+                    stream,
+                });
+
+                if ( stream ) {
+                    const stream = new PassThrough();
+                    const retval = new TypedValue({
+                        $: 'stream',
+                        content_type: 'application/x-ndjson',
+                        chunked: true,
+                    }, stream);
+                    (async () => {
+                        for await ( const chunk of completion ) {
+                            if ( chunk.choices.length < 1 ) continue;
+                            if ( chunk.choices[0].finish_reason ) {
+                                stream.end();
+                                break;
+                            }
+                            if ( nou(chunk.choices[0].delta.content) ) continue;
+                            const str = JSON.stringify({
+                                text: chunk.choices[0].delta.content
+                            });
+                            stream.write(str + '\n');
+                        }
+                    })();
+                    return retval;
+                }
+                
+                return completion.choices[0];
+            }
+        }
+    };
+}
+
+module.exports = {
+    GroqAIService,
+};

src/backend/src/modules/puterai/PuterAIModule.js

@@ -43,6 +43,11 @@ class PuterAIModule extends AdvancedBase {
             const { MistralAIService } = require('./MistralAIService');
             services.registerService('mistral', MistralAIService);
         }
+        
+        if ( !! config?.services?.['groq'] ) {
+            const { GroqAIService } = require('./GroqAIService');
+            services.registerService('groq', GroqAIService);
+        }
     }
 }
 

src/puter-js/src/modules/AI.js

@@ -229,6 +229,9 @@ class AI{
         if ( options.model === 'mistral' ) {
             options.model = 'mistral-large-latest';
         }
+        if ( options.model === 'groq' ) {
+            options.model = 'llama3-8b-8192';
+        }
 
         // map model to the appropriate driver
         if (!options.model || options.model === 'gpt-4o' || options.model === 'gpt-4o-mini') {
@@ -239,6 +242,21 @@ class AI{
             driver = 'together-ai';
         }else if(options.model === 'mistral-large-latest' || options.model === 'codestral-latest'){
             driver = 'mistral';
+        }else if([
+            "distil-whisper-large-v3-en",
+            "gemma2-9b-it",
+            "gemma-7b-it",
+            "llama-3.1-70b-versatile",
+            "llama-3.1-8b-instant",
+            "llama3-70b-8192",
+            "llama3-8b-8192",
+            "llama3-groq-70b-8192-tool-use-preview",
+            "llama3-groq-8b-8192-tool-use-preview",
+            "llama-guard-3-8b",
+            "mixtral-8x7b-32768",
+            "whisper-large-v3"
+        ].includes(options.model)) {
+            driver = 'groq';
         }
 
         // stream flag from settings