Merge c5f0b1d47d into f2f06fcf3f

README.md

@@ -12,7 +12,17 @@ Do not connect this bot to public servers with coding enabled. This project allo
 
 - [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) (up to v1.21.1, recommend v1.20.4)
 - [Node.js Installed](https://nodejs.org/) (at least v14)
-- One of these: [OpenAI API Key](https://openai.com/blog/openai-api) | [Gemini API Key](https://aistudio.google.com/app/apikey) | [Anthropic API Key](https://docs.anthropic.com/claude/docs/getting-access-to-claude) | [Replicate API Key](https://replicate.com/) | [Hugging Face API Key](https://huggingface.co/) | [Groq API Key](https://console.groq.com/keys) | [Ollama Installed](https://ollama.com/download). | [Mistral API Key](https://docs.mistral.ai/getting-started/models/models_overview/) | [Qwen API Key [Intl.]](https://www.alibabacloud.com/help/en/model-studio/developer-reference/get-api-key)/[[cn]](https://help.aliyun.com/zh/model-studio/getting-started/first-api-call-to-qwen?) | [Novita AI API Key](https://novita.ai/settings?utm_source=github_mindcraft&utm_medium=github_readme&utm_campaign=link#key-management) |
+- One of these:
+  - [OpenAI API Key](https://openai.com/blog/openai-api)
+  - [Gemini API Key](https://aistudio.google.com/app/apikey)
+  - [Anthropic API Key](https://docs.anthropic.com/claude/docs/getting-access-to-claude)
+  - [Replicate API Key](https://replicate.com/)
+  - [Hugging Face API Key](https://huggingface.co/)
+  - [Groq API Key](https://console.groq.com/keys)
+  - [Ollama Installed](https://ollama.com/download)
+  - [Mistral API Key](https://docs.mistral.ai/getting-started/models/models_overview/)
+  - [Qwen API Key [Intl.]](https://www.alibabacloud.com/help/en/model-studio/developer-reference/get-api-key)/[[cn]](https://help.aliyun.com/zh/model-studio/getting-started/first-api-call-to-qwen?)
+  - [Novita AI API Key](https://novita.ai/settings?utm_source=github_mindcraft&utm_medium=github_readme&utm_campaign=link#key-management)
 
 ## Install and Run
 
@@ -63,6 +73,7 @@ You can configure the agent's name, model, and prompts in their profile like `an
 | `openrouter` | `OPENROUTER_API_KEY` | `openrouter/anthropic/claude-3.5-sonnet` | [docs](https://openrouter.ai/models) |
 | `glhf.chat` | `GHLF_API_KEY` | `glhf/hf:meta-llama/Llama-3.1-405B-Instruct` | [docs](https://glhf.chat/user-settings/api) |
 | `hyperbolic` | `HYPERBOLIC_API_KEY` | `hyperbolic/deepseek-ai/DeepSeek-V3` | [docs](https://docs.hyperbolic.xyz/docs/getting-started) |
+| `pollinations` | n/a | `pollinations/openai-large` | [docs](https://github.com/pollinations/pollinations/blob/master/APIDOCS.md) |
 | `vllm` | n/a | `vllm/llama3` | n/a |
 
 If you use Ollama, to install the models used by default (generation and embedding), execute the following terminal command:
@@ -139,6 +150,12 @@ You can pass a string or an object for these fields. A model object must specify
   "api": "openai",
   "url": "https://api.openai.com/v1/",
   "model": "text-embedding-ada-002"
+},
+"speak_model": {
+  "api": "pollinations",
+  "url": "https://text.pollinations.ai/openai",
+  "model": "openai-audio",
+  "voice": "echo"
 }
 
 ```

profiles/defaults/_default.json

@@ -11,6 +11,8 @@
 
     "image_analysis": "You are a Minecraft bot named $NAME that has been given a screenshot of your current view. Analyze and summarize the view; describe terrain, blocks, entities, structures, and notable features. Focus on details relevant to the conversation. Note: the sky is always blue regardless of weather or time, dropped items are small pink cubes, and blocks below y=0 do not render. Be extremely concise and correct, respond only with your analysis, not conversationally. $STATS",
 
+    "speak_model": "pollinations/openai-audio/echo",
+    
     "modes": {
         "self_preservation": true,
         "unstuck": true,

settings.js

@@ -28,8 +28,12 @@ const settings = {
     "load_memory": false, // load memory from previous session
     "init_message": "Respond with hello world and your name", // sends to all on spawn
     "only_chat_with": [], // users that the bots listen to and send general messages to. if empty it will chat publicly
-    "speak": false, // allows all bots to speak through system text-to-speech. works on windows, mac, on linux you need to `apt install espeak`
+    
-    "language": "en", // translate to/from this language. Supports these language names: https://cloud.google.com/translate/docs/languages
+    "speak": true,
+    // allows all bots to speak through text-to-speech. format: {provider}/{model}/{voice}. if set to "system" it will use system text-to-speech, which works on windows and mac, but on linux you need to `apt install espeak`.
+    // specify speech model inside each profile - so that you can have each bot with different voices ;)
+    
+    "language": "en", // translate to/from this language. NOT text-to-speech language. Supports these language names: https://cloud.google.com/translate/docs/languages
     "show_bot_views": false, // show bot's view in browser at localhost:3000, 3001...
 
     "allow_insecure_coding": false, // allows newAction command and model can write/run code on your computer. enable at own risk

src/agent/agent.js

@@ -389,9 +389,9 @@ export class Agent {
             }
         }
         else {
-	    if (settings.speak) {
+            if (settings.speak) {
-            say(to_translate);
+                say(to_translate, this.prompter.profile.speak_model);
-	    }
+            }
             this.bot.chat(message);
         }
     }

src/agent/speak.js

@@ -1,43 +1,87 @@
-import { exec } from 'child_process';
+import { exec, spawn } from 'child_process';
+import { sendAudioRequest } from '../models/pollinations.js';
 
 let speakingQueue = [];
 let isSpeaking = false;
 
-export function say(textToSpeak) {
+export function say(text, speak_model) {
-  speakingQueue.push(textToSpeak);
+  speakingQueue.push([text, speak_model]);
-  if (!isSpeaking) {
+  if (!isSpeaking) processQueue();
-    processQueue();
-  }
 }
 
-function processQueue() {
+async function processQueue() {
   if (speakingQueue.length === 0) {
     isSpeaking = false;
     return;
   }
-
   isSpeaking = true;
-  const textToSpeak = speakingQueue.shift();
+  const [txt, speak_model] = speakingQueue.shift();
-  const isWin = process.platform === "win32";
-  const isMac = process.platform === "darwin";
 
-  let command;
+  const isWin = process.platform === 'win32';
+  const isMac = process.platform === 'darwin';
+  const model = speak_model || 'pollinations/openai-audio/echo';
+
+  if (model === 'system') {
+    // system TTS
+    const cmd = isWin
+      ? `powershell -NoProfile -Command "Add-Type -AssemblyName System.Speech; \
+$s=New-Object System.Speech.Synthesis.SpeechSynthesizer; $s.Rate=2; \
+$s.Speak('${txt.replace(/'/g,"''")}'); $s.Dispose()"`
+      : isMac
+      ? `say "${txt.replace(/"/g,'\\"')}"`
+      : `espeak "${txt.replace(/"/g,'\\"')}"`;
+
+    exec(cmd, err => {
+      if (err) console.error('TTS error', err);
+      processQueue();
+    });
 
-  if (isWin) {
-    command = `powershell -Command "Add-Type -AssemblyName System.Speech; $s = New-Object System.Speech.Synthesis.SpeechSynthesizer; $s.Rate = 2; $s.Speak(\\"${textToSpeak}\\"); $s.Dispose()"`;
-  } else if (isMac) {
-    command = `say "${textToSpeak}"`;
   } else {
-    command = `espeak "${textToSpeak}"`;
+    // remote audio provider
-  }
+    let prov, mdl, voice, url;
-
+    if (typeof model === "string") {
-  exec(command, (error, stdout, stderr) => {
+      [prov, mdl, voice] = model.split('/');
-    if (error) {
+      url = "https://text.pollinations.ai/openai";
-      console.error(`Error: ${error.message}`);
+    } else {
-      console.error(`${error.stack}`);
+      prov = model.api;
-    } else if (stderr) {
+      mdl = model.model;
-      console.error(`Error: ${stderr}`);
+      voice = model.voice;
+      url = model.url || "https://text.pollinations.ai/openai";
     }
-    processQueue(); // Continue with the next message in the queue
+    if (prov !== 'pollinations') throw new Error(`Unknown provider: ${prov}`);
-  });
+
+    try {
+      let audioData = await sendAudioRequest(txt, mdl, voice, url);
+      if (!audioData) {
+        audioData = "SUQzBAAAAAAAI1RTU0UAAAAPAAADTGF2ZjU5LjI3LjEwMAAAAAAAAAAAAAAA/+NAwAAAAAAAAAAAAEluZm8AAAAPAAAAAAAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAExhdmM1OS4zNwAAAAAAAAAAAAAAAAAAAAAAAAAAAADQAAAeowAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==";
+        // ^ 0 second silent audio clip
+      }
+
+      if (isWin) {
+        const ps = `
+          Add-Type -AssemblyName presentationCore;
+          $p=New-Object System.Windows.Media.MediaPlayer;
+          $p.Open([Uri]::new("data:audio/mp3;base64,${audioData}"));
+          $p.Play();
+          Start-Sleep -Seconds [math]::Ceiling($p.NaturalDuration.TimeSpan.TotalSeconds);
+        `;
+        spawn('powershell', ['-NoProfile','-Command', ps], {
+          stdio: 'ignore', detached: true
+        }).unref();
+        processQueue();
+
+      } else {
+        const player = spawn('ffplay', ['-nodisp','-autoexit','pipe:0'], {
+          stdio: ['pipe','ignore','ignore']
+        });
+        player.stdin.write(Buffer.from(audioData, 'base64'));
+        player.stdin.end();
+        player.on('exit', processQueue);
+      }
+
+    } catch (e) {
+      console.error('Audio error', e);
+      processQueue();
+    }
+  }
 }

src/models/pollinations.js

@@ -0,0 +1,110 @@
+import { strictFormat } from "../utils/text.js";
+
+export class Pollinations {
+    // models: https://text.pollinations.ai/models
+    constructor(model_name, url, params) {
+        this.model_name = model_name;
+        this.params = params;
+        this.url = url || "https://text.pollinations.ai/openai";
+    }
+
+    async sendRequest(turns, systemMessage) {
+        let messages = [{'role': 'system', 'content': systemMessage}].concat(turns);
+
+        const payload = {
+            model: this.model_name || "openai-large",
+            messages: strictFormat(messages),
+            seed: Math.floor( Math.random() * (99999) ),
+            referrer: "mindcraft",
+            ...(this.params || {})
+        };
+
+        let res = null;
+
+        try {
+            console.log(`Awaiting pollinations response from model`, this.model_name);
+            const response = await fetch(this.url, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json"
+                },
+                body: JSON.stringify(payload)
+            });
+            if (!response.ok) {
+                console.error(`Failed to receive response. Status`, response.status, (await response.text()));
+                res = "My brain disconnected, try again.";
+            } else {
+                const result = await response.json();
+                res = result.choices[0].message.content;
+            }
+        } catch (err) {
+            console.error(`Failed to receive response.`, err || err.message);
+            res = "My brain disconnected, try again.";
+        }
+        return res;
+    }
+
+    async sendVisionRequest(messages, systemMessage, imageBuffer) {
+        const imageMessages = [...messages];
+        imageMessages.push({
+            role: "user",
+            content: [
+                { type: "text", text: systemMessage },
+                {
+                    type: "image_url",
+                    image_url: {
+                        url: `data:image/jpeg;base64,${imageBuffer.toString('base64')}`
+                    }
+                }
+            ]
+        });
+
+        return this.sendRequest(imageMessages, systemMessage)
+    }
+}
+
+export async function sendAudioRequest(text, model, voice, url) {
+    const payload = {
+        model: model,
+        modalities: ["text", "audio"],
+        audio: {
+            voice: voice,
+            format: "mp3",
+        },
+        messages: [
+            {
+                role: "developer",
+                content: "You are an AI that echoes. Your sole function is to repeat back everything the user says to you exactly as it is written. This includes punctuation, grammar, language, and text formatting. Do not add, remove, or alter anything in the user's input in any way. Respond only with an exact duplicate of the user’s query."
+                // this is required because pollinations attempts to send an AI response to the text instead of just saying the text.
+            },
+            {
+                role: "user",
+                content: text
+            }
+        ]
+    }
+
+    let audioData = null;
+
+    try {
+        const response = await fetch(url, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify(payload)
+        })
+
+        if (!response.ok) {
+            console.error("Failed to get text transcription. Status", response.status, (await response.text()))
+            return null;
+        }
+
+        const result = await response.json();
+        audioData = result.choices[0].message.audio.data;
+        return audioData;
+    } catch (err) {
+        console.error("TTS fetch failed:", err);
+        return null;
+    }
+}

src/models/prompter.js

@@ -21,6 +21,7 @@ import { DeepSeek } from './deepseek.js';
 import { Hyperbolic } from './hyperbolic.js';
 import { GLHF } from './glhf.js';
 import { OpenRouter } from './openrouter.js';
+import { Pollinations } from './pollinations.js';
 import { VLLM } from './vllm.js';
 import { promises as fs } from 'fs';
 import path from 'path';
@@ -140,6 +141,8 @@ export class Prompter {
                 profile.api = 'openrouter'; // must do first because shares names with other models
             else if (profile.model.includes('ollama/'))
                 profile.api = 'ollama'; // also must do early because shares names with other models
+            else if (profile.model.includes('pollinations/'))
+                profile.api = 'pollinations'; // also shares some model names like llama
             else if (profile.model.includes('gemini'))
                 profile.api = 'google';
             else if (profile.model.includes('vllm/'))
@@ -207,6 +210,8 @@ export class Prompter {
             model = new DeepSeek(profile.model, profile.url, profile.params);
         else if (profile.api === 'openrouter')
             model = new OpenRouter(profile.model.replace('openrouter/', ''), profile.url, profile.params);
+        else if (profile.api === 'pollinations')
+            model = new Pollinations(profile.model.replace('pollinations/', ''), profile.url, profile.params);
         else if (profile.api === 'vllm')
             model = new VLLM(profile.model.replace('vllm/', ''), profile.url, profile.params);
         else