Merge remote-tracking branch 'refs/remotes/upstream/main' into Tasks-more-relevant-docs-and-code-exception-fixes

# Conflicts:
#	src/models/prompter.js
#	src/models/qwen.js

README.md

@@ -1,12 +1,11 @@
 # Mindcraft 🧠⛏️
 
-Crafting minds for Minecraft with LLMs and Mineflayer!
+Crafting minds for Minecraft with LLMs and [Mineflayer!](https://prismarinejs.github.io/mineflayer/#/)
 
 [FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) | [Discord Support](https://discord.gg/mp73p35dzC) | [Blog Post](https://kolbynottingham.com/mindcraft/) | [Contributor TODO](https://github.com/users/kolbytn/projects/1)
 
 
-#### ‼️Warning‼️
+> [!WARNING]
-
 Do not connect this bot to public servers with coding enabled. This project allows an LLM to write/execute code on your computer. While the code is sandboxed, it is still vulnerable to injection attacks on public servers. Code writing is disabled by default, you can enable it by setting `allow_insecure_coding` to `true` in `settings.js`. We strongly recommend running with additional layers of security such as docker containers. Ye be warned.
 
 ## Requirements
@@ -42,7 +41,7 @@ You can configure the agent's name, model, and prompts in their profile like `an
 | OpenAI | `OPENAI_API_KEY` | `gpt-4o-mini` | [docs](https://platform.openai.com/docs/models) |
 | Google | `GEMINI_API_KEY` | `gemini-pro` | [docs](https://ai.google.dev/gemini-api/docs/models/gemini) |
 | Anthropic | `ANTHROPIC_API_KEY` | `claude-3-haiku-20240307` | [docs](https://docs.anthropic.com/claude/docs/models-overview) |
-| Replicate | `REPLICATE_API_KEY` | `meta/meta-llama-3-70b-instruct` | [docs](https://replicate.com/collections/language-models) |
+| Replicate | `REPLICATE_API_KEY` | `replicate/meta/meta-llama-3-70b-instruct` | [docs](https://replicate.com/collections/language-models) |
 | Ollama (local) | n/a | `llama3` | [docs](https://ollama.com/library) |
 | Groq | `GROQCLOUD_API_KEY` | `groq/mixtral-8x7b-32768` | [docs](https://console.groq.com/docs/models) |
 | Hugging Face | `HUGGINGFACE_API_KEY` | `huggingface/mistralai/Mistral-Nemo-Instruct-2407` | [docs](https://huggingface.co/models) |
@@ -63,7 +62,8 @@ To connect to online servers your bot will need an official Microsoft/Minecraft
 
 // rest is same...
 ```
-‼️ The bot's name in the profile.json must exactly match the Minecraft profile name! Otherwise the bot will spam talk to itself.
+> [!CAUTION]
+> The bot's name in the profile.json must exactly match the Minecraft profile name! Otherwise the bot will spam talk to itself.
 
 To use different accounts, Mindcraft will connect with the account that the Minecraft launcher is currently using. You can switch accounts in the launcer, then run `node main.js`, then switch to your main account after the bot has connected.
 
@@ -105,39 +105,37 @@ node main.js --profiles ./profiles/andy.json ./profiles/jill.json
 
 ### Model Specifications
 
-LLM backends can be specified as simply as `"model": "gpt-3.5-turbo"`. However, for both the chat model and the embedding model, the bot profile can specify the below attributes:
+LLM models can be specified as simply as `"model": "gpt-4o"`. However, you can specify different models for chat, coding, and embeddings. 
+You can pass a string or an object for these fields. A model object must specify an `api`, and optionally a `model`, `url`, and additional `params`.
 
 ```json
 "model": {
   "api": "openai",
+  "model": "gpt-4o",
   "url": "https://api.openai.com/v1/",
-  "model": "gpt-3.5-turbo"
+  "params": {
+    "max_tokens": 1000,
+    "temperature": 1
+  }
+},
+"code_model": {
+  "api": "openai",
+  "model": "gpt-4",
+  "url": "https://api.openai.com/v1/"
 },
 "embedding": {
   "api": "openai",
   "url": "https://api.openai.com/v1/",
   "model": "text-embedding-ada-002"
 }
+
 ```
 
-The model parameter accepts either a string or object. If a string, it should specify the model to be used. The api and url will be assumed. If an object, the api field must be specified. Each api has a default model and url, so those fields are optional.
+`model` is used for chat, `code_model` is used for newAction coding, and `embedding` is used to embed text for example selection. If `code_model` is not specified, then it will use `model` for coding.
 
-If the embedding field is not specified, then it will use the default embedding method for the chat model's api (Note that anthropic has no embedding model). The embedding parameter can also be a string or object. If a string, it should specify the embedding api and the default model and url will be used. If a valid embedding is not specified and cannot be assumed, then word overlap will be used to retrieve examples instead.
+All apis have default models and urls, so those fields are optional. Note some apis have no embedding model, so they will default to word overlap to retrieve examples. 
 
-Thus, all the below specifications are equivalent to the above example:
+The `params` field is optional and can be used to specify additional parameters for the model. It accepts any key-value pairs supported by the api. Is not supported for embedding models.
-
-```json
-"model": "gpt-3.5-turbo"
-```
-```json
-"model": {
-  "api": "openai"
-}
-```
-```json
-"model": "gpt-3.5-turbo",
-"embedding": "openai"
-```
 
 ## Patches
 

patches/mineflayer-collectblock+1.4.1.patch

@@ -1,35 +0,0 @@
-diff --git a/node_modules/mineflayer-collectblock/lib/CollectBlock.js b/node_modules/mineflayer-collectblock/lib/CollectBlock.js
-index 2c11e8c..bb49c11 100644
---- a/node_modules/mineflayer-collectblock/lib/CollectBlock.js
-+++ b/node_modules/mineflayer-collectblock/lib/CollectBlock.js
-@@ -77,10 +77,11 @@ function mineBlock(bot, block, options) {
-         }
-         yield bot.tool.equipForBlock(block, equipToolOptions);
-         // @ts-expect-error
--        if (!block.canHarvest(bot.heldItem)) {
-+        if (bot.heldItem !== null && !block.canHarvest(bot.heldItem.type)) {
-             options.targets.removeTarget(block);
-             return;
-         }
-+        
-         const tempEvents = new TemporarySubscriber_1.TemporarySubscriber(bot);
-         tempEvents.subscribeTo('itemDrop', (entity) => {
-             if (entity.position.distanceTo(block.position.offset(0.5, 0.5, 0.5)) <= 0.5) {
-@@ -92,7 +93,7 @@ function mineBlock(bot, block, options) {
-             // Waiting for items to drop
-             yield new Promise(resolve => {
-                 let remainingTicks = 10;
--                tempEvents.subscribeTo('physicTick', () => {
-+                tempEvents.subscribeTo('physicsTick', () => {
-                     remainingTicks--;
-                     if (remainingTicks <= 0) {
-                         tempEvents.cleanup();
-@@ -195,6 +196,8 @@ class CollectBlock {
-                 throw (0, Util_1.error)('UnresolvedDependency', 'The mineflayer-collectblock plugin relies on the mineflayer-tool plugin to run!');
-             }
-             if (this.movements != null) {
-+                this.movements.dontMineUnderFallingBlock = false;
-+                this.movements.dontCreateFlow = false;
-                 this.bot.pathfinder.setMovements(this.movements);
-             }
-             if (!optionsFull.append)

profiles/gpt.json

@@ -1,5 +1,10 @@
 {
     "name": "gpt",
 
-    "model": "gpt-4o"
+    "model": {
+        "model": "gpt-4o",
+        "params": {
+            "temperature": 0.5
+        }
+    }
 }

src/agent/agent.js

@@ -1,6 +1,6 @@
 import { History } from './history.js';
 import { Coder } from './coder.js';
-import { Prompter } from './prompter.js';
+import { Prompter } from '../models/prompter.js';
 import { initModes } from './modes.js';
 import { initBot } from '../utils/mcdata.js';
 import { containsCommand, commandExists, executeCommand, truncCommandMessage, isAction, blacklistCommands } from './commands/index.js';
@@ -100,11 +100,9 @@ export class Agent {
             });
         } catch (error) {
             // Ensure we're not losing error details
-            console.error('Agent start failed with error:', {
+            console.error('Agent start failed with error')
-                message: error.message || 'No error message',
+            console.error(error)
-                stack: error.stack || 'No stack trace',
+
-                error: error
-            });
             throw error; // Re-throw with preserved details
         }
     }

src/agent/commands/index.js

@@ -160,7 +160,7 @@ export function parseCommandMessage(message) {
                 suppressNoDomainWarning = true; //Don't spam console. Only give the warning once.
             }
         } else if(param.type === 'BlockName') { //Check that there is a block with this name
-            if(getBlockId(arg) == null) return  `Invalid block type: ${arg}.`
+            if(getBlockId(arg) == null && arg !== 'air') return  `Invalid block type: ${arg}.`
         } else if(param.type === 'ItemName') { //Check that there is an item with this name
             if(getItemId(arg) == null) return `Invalid item type: ${arg}.`
         }

src/agent/commands/queries.js

@@ -178,6 +178,42 @@ export const queryList = [
             return "Saved place names: " + agent.memory_bank.getKeys();
         }
     },
+    {
+        name: '!getCraftingPlan',
+        description: "Provides a comprehensive crafting plan for a specified item. This includes a breakdown of required ingredients, the exact quantities needed, and an analysis of missing ingredients or extra items needed based on the bot's current inventory.",
+        params: {
+            targetItem: { 
+                type: 'string', 
+                description: 'The item that we are trying to craft' 
+            },
+            quantity: { 
+                type: 'int',
+                description: 'The quantity of the item that we are trying to craft',
+                optional: true,
+                domain: [1, Infinity, '[)'], // Quantity must be at least 1,
+                default: 1
+            }
+        },
+        perform: function (agent, targetItem, quantity = 1) {
+            let bot = agent.bot;
+
+            // Fetch the bot's inventory
+            const curr_inventory = world.getInventoryCounts(bot); 
+            const target_item = targetItem;
+            let existingCount = curr_inventory[target_item] || 0;
+            let prefixMessage = '';
+            if (existingCount > 0) {
+                curr_inventory[target_item] -= existingCount;
+                prefixMessage = `You already have ${existingCount} ${target_item} in your inventory. If you need to craft more,\n`;
+            }
+
+            // Generate crafting plan
+            let craftingPlan = mc.getDetailedCraftingPlan(target_item, quantity, curr_inventory);
+            craftingPlan = prefixMessage + craftingPlan;
+            console.log(craftingPlan);
+            return pad(craftingPlan);
+        },
+    },
     {
         name: '!help',
         description: 'Lists all available commands and their descriptions.',

src/agent/library/skills.js

@@ -79,7 +79,7 @@ export async function craftRecipe(bot, itemName, num=1) {
         }
     }
     if (!recipes || recipes.length === 0) {
-        log(bot, `You do not have the resources to craft a ${itemName}. It requires: ${Object.entries(mc.getItemCraftingRecipes(itemName)[0]).map(([key, value]) => `${key}: ${value}`).join(', ')}.`);
+        log(bot, `You do not have the resources to craft a ${itemName}. It requires: ${Object.entries(mc.getItemCraftingRecipes(itemName)[0][0]).map(([key, value]) => `${key}: ${value}`).join(', ')}.`);
         if (placedTable) {
             await collectBlock(bot, 'crafting_table', 1);
         }
@@ -1275,8 +1275,14 @@ export async function tillAndSow(bot, x, y, z, seedType=null) {
     let block = bot.blockAt(new Vec3(x, y, z));
 
     if (bot.modes.isOn('cheat')) {
-        placeBlock(bot, x, y, z, 'farmland');
+        let to_remove = ['_seed', '_seeds'];
-        placeBlock(bot, x, y+1, z, seedType);
+        for (let remove of to_remove) {
+            if (seedType.endsWith(remove)) {
+                seedType = seedType.replace(remove, '');
+            }
+        }
+        placeBlock(bot, 'farmland', x, y, z);
+        placeBlock(bot, seedType, x, y+1, z);
         return true;
     }
 

src/agent/npc/item_goal.js

@@ -204,7 +204,7 @@ class ItemWrapper {
     }
 
     createChildren() {
-        let recipes = mc.getItemCraftingRecipes(this.name);
+        let recipes = mc.getItemCraftingRecipes(this.name).map(([recipe, craftedCount]) => recipe);
         if (recipes) {
             for (let recipe of recipes) {
                 let includes_blacklisted = false;

src/agent/self_prompter.js

@@ -38,7 +38,7 @@ export class SelfPrompter {
         let no_command_count = 0;
         const MAX_NO_COMMAND = 3;
         while (!this.interrupt) {
-            const msg = `You are self-prompting with the goal: '${this.prompt}'. Your next response MUST contain a command !withThisSyntax. Respond:`;
+            const msg = `You are self-prompting with the goal: '${this.prompt}'. Your next response MUST contain a command with this syntax: !commandName. Respond:`;
             
             let used_command = await this.agent.handleMessage('system', msg, -1);
             if (!used_command) {

src/agent/tasks.js

@@ -109,11 +109,11 @@ export class Task {
         await new Promise((resolve) => setTimeout(resolve, 500));
     
         if (this.data.agent_count > 1) {
-            var initial_inventory = this.data.initial_inventory[this.agent.count_id.toString()];
+            let initial_inventory = this.data.initial_inventory[this.agent.count_id.toString()];
             console.log("Initial inventory:", initial_inventory);
         } else if (this.data) {
             console.log("Initial inventory:", this.data.initial_inventory);
-            var initial_inventory = this.data.initial_inventory;
+            let initial_inventory = this.data.initial_inventory;
         }
     
         if ("initial_inventory" in this.data) {

src/models/claude.js

@@ -3,8 +3,9 @@ import { strictFormat } from '../utils/text.js';
 import { getKey } from '../utils/keys.js';
 
 export class Claude {
-    constructor(model_name, url) {
+    constructor(model_name, url, params) {
         this.model_name = model_name;
+        this.params = params;
 
         let config = {};
         if (url)
@@ -20,13 +21,16 @@ export class Claude {
         let res = null;
         try {
             console.log('Awaiting anthropic api response...')
-            // console.log('Messages:', messages);
+            if (!this.params.max_tokens) {
+                this.params.max_tokens = 4096;
+            }
             const resp = await this.anthropic.messages.create({
                 model: this.model_name || "claude-3-sonnet-20240229",
                 system: systemMessage,
-                max_tokens: 2048,
                 messages: messages,
+                ...(this.params || {})
             });
+
             console.log('Received.')
             res = resp.content[0].text;
         }

src/models/deepseek.js

@@ -3,8 +3,9 @@ import { getKey, hasKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
 
 export class DeepSeek {
-    constructor(model_name, url) {
+    constructor(model_name, url, params) {
         this.model_name = model_name;
+        this.params = params;
 
         let config = {};
 
@@ -23,6 +24,7 @@ export class DeepSeek {
             model: this.model_name || "deepseek-chat",
             messages,
             stop: stop_seq,
+            ...(this.params || {})
         };
 
         let res = null;

src/models/gemini.js

@@ -1,10 +1,11 @@
 import { GoogleGenerativeAI } from '@google/generative-ai';
-import { toSinglePrompt } from '../utils/text.js';
+import { toSinglePrompt, strictFormat } from '../utils/text.js';
 import { getKey } from '../utils/keys.js';
 
 export class Gemini {
-    constructor(model_name, url) {
+    constructor(model_name, url, params) {
         this.model_name = model_name;
+        this.params = params;
         this.url = url;
         this.safetySettings = [
             {
@@ -34,28 +35,46 @@ export class Gemini {
 
     async sendRequest(turns, systemMessage) {
         let model;
+        const modelConfig = {
+            model: this.model_name || "gemini-1.5-flash",
+            // systemInstruction does not work bc google is trash
+        };
+        
         if (this.url) {
             model = this.genAI.getGenerativeModel(
-                { model: this.model_name || "gemini-1.5-flash" },
+                modelConfig,
                 { baseUrl: this.url },
                 { safetySettings: this.safetySettings }
             );
         } else {
             model = this.genAI.getGenerativeModel(
-                { model: this.model_name || "gemini-1.5-flash" },
+                modelConfig,
                 { safetySettings: this.safetySettings }
             );
         }
 
-        const stop_seq = '***';
-        const prompt = toSinglePrompt(turns, systemMessage, stop_seq, 'model');
         console.log('Awaiting Google API response...');
-        const result = await model.generateContent(prompt);
+
+        turns.unshift({ role: 'system', content: systemMessage });
+        turns = strictFormat(turns);
+        let contents = [];
+        for (let turn of turns) {
+            contents.push({
+                role: turn.role === 'assistant' ? 'model' : 'user',
+                parts: [{ text: turn.content }]
+            });
+        }
+
+        const result = await model.generateContent({
+            contents,
+            generationConfig: {
+                ...(this.params || {})
+            }
+        });
         const response = await result.response;
         const text = response.text();
         console.log('Received.');
-        if (!text.includes(stop_seq)) return text;
+
-        const idx = text.indexOf(stop_seq);
         return text.slice(0, idx);
     }
 

src/models/gpt.js

@@ -3,8 +3,9 @@ import { getKey, hasKey } from '../utils/keys.js';
 import { strictFormat } from '../utils/text.js';
 
 export class GPT {
-    constructor(model_name, url) {
+    constructor(model_name, url, params) {
         this.model_name = model_name;
+        this.params = params;
 
         let config = {};
         if (url)
@@ -25,6 +26,7 @@ export class GPT {
             model: this.model_name || "gpt-3.5-turbo",
             messages,
             stop: stop_seq,
+            ...(this.params || {})
         };
         if (this.model_name.includes('o1')) {
             pack.messages = strictFormat(messages);
@@ -32,8 +34,9 @@ export class GPT {
         }
 
         let res = null;
+
         try {
-            console.log('Awaiting openai api response...')
+            console.log('Awaiting openai api response from model', this.model_name)
             // console.log('Messages:', messages);
             let completion = await this.openai.chat.completions.create(pack);
             if (completion.choices[0].finish_reason == 'length')

src/models/grok.js

@@ -3,8 +3,10 @@ import { getKey } from '../utils/keys.js';
 
 // xAI doesn't supply a SDK for their models, but fully supports OpenAI and Anthropic SDKs
 export class Grok {
-    constructor(model_name, url) {
+    constructor(model_name, url, params) {
         this.model_name = model_name;
+        this.url = url;
+        this.params = params;
 
         let config = {};
         if (url)
@@ -23,7 +25,8 @@ export class Grok {
         const pack = {
             model: this.model_name || "grok-beta",
             messages,
-            stop: [stop_seq]
+            stop: [stop_seq],
+            ...(this.params || {})
         };
 
         let res = null;

src/models/groq.js

@@ -4,12 +4,13 @@ import { getKey } from '../utils/keys.js';
 
 // Umbrella class for Mixtral, LLama, Gemma...
 export class GroqCloudAPI {
-    constructor(model_name, url, max_tokens=16384) {
+    constructor(model_name, url, params) {
         this.model_name = model_name;
         this.url = url;
-        this.max_tokens = max_tokens;
+        this.params = params;
         // ReplicateAPI theft :3
         if (this.url) {
+
             console.warn("Groq Cloud has no implementation for custom URLs. Ignoring provided URL.");
         }
         this.groq = new Groq({ apiKey: getKey('GROQCLOUD_API_KEY') });
@@ -20,14 +21,15 @@ export class GroqCloudAPI {
         let res = null;
         try {
             console.log("Awaiting Groq response...");
+            if (!this.params.max_tokens) {
+                this.params.max_tokens = 16384;
+            }
             let completion = await this.groq.chat.completions.create({
                 "messages": messages,
                 "model": this.model_name || "mixtral-8x7b-32768",
-                "temperature": 0.2,
-                "max_tokens": this.max_tokens, // maximum token limit, differs from model to model
-                "top_p": 1,
                 "stream": true,
-                "stop": stop_seq // "***"
+                "stop": stop_seq,
+                ...(this.params || {})
             });
 
             let temp_res = "";

src/models/huggingface.js

@@ -3,9 +3,10 @@ import {getKey} from '../utils/keys.js';
 import {HfInference} from "@huggingface/inference";
 
 export class HuggingFace {
-    constructor(model_name, url) {
+    constructor(model_name, url, params) {
         this.model_name = model_name.replace('huggingface/','');
         this.url = url;
+        this.params = params;
 
         if (this.url) {
             console.warn("Hugging Face doesn't support custom urls!");
@@ -25,7 +26,8 @@ export class HuggingFace {
             console.log('Awaiting Hugging Face API response...');
             for await (const chunk of this.huggingface.chatCompletionStream({
                 model: model_name,
-                messages: [{ role: "user", content: input }]
+                messages: [{ role: "user", content: input }],
+                ...(this.params || {})
             })) {
                 res += (chunk.choices[0]?.delta?.content || "");
             }

src/models/local.js

@@ -1,8 +1,9 @@
 import { strictFormat } from '../utils/text.js';
 
 export class Local {
-    constructor(model_name, url) {
+    constructor(model_name, url, params) {
         this.model_name = model_name;
+        this.params = params;
         this.url = url || 'http://127.0.0.1:11434';
         this.chat_endpoint = '/api/chat';
         this.embedding_endpoint = '/api/embeddings';
@@ -15,7 +16,12 @@ export class Local {
         let res = null;
         try {
             console.log(`Awaiting local response... (model: ${model})`)
-            res = await this.send(this.chat_endpoint, {model: model, messages: messages, stream: false});
+            res = await this.send(this.chat_endpoint, {
+                model: model, 
+                messages: messages, 
+                stream: false,
+                ...(this.params || {})
+            });
             if (res)
                 res = res['message']['content'];
         }

src/models/mistral.js

@@ -5,10 +5,13 @@ import { strictFormat } from '../utils/text.js';
 export class Mistral {
     #client;
 
-    constructor(model_name, url) {
+    constructor(model_name, url, params) {
+        this.model_name = model_name;
+        this.params = params;
 
         if (typeof url === "string") {
             console.warn("Mistral does not support custom URL's, ignoring!");
+
         }
 
         if (!getKey("MISTRAL_API_KEY")) {
@@ -22,8 +25,6 @@ export class Mistral {
         );
 
         
-        this.model_name = model_name;
-
         // Prevents the following code from running when model not specified
         if (typeof this.model_name === "undefined") return;
 
@@ -49,6 +50,7 @@ export class Mistral {
             const response  = await this.#client.chat.complete({
                 model,
                 messages,
+                ...(this.params || {})
             });
 
             result = response.choices[0].message.content;

src/models/novita.js

@@ -1,11 +1,14 @@
 import OpenAIApi from 'openai';
 import { getKey } from '../utils/keys.js';
+import { strictFormat } from '../utils/text.js';
 
 // llama, mistral
 export class Novita {
-	constructor(model_name, url) {
+	constructor(model_name, url, params) {
     this.model_name = model_name.replace('novita/', '');
     this.url = url || 'https://api.novita.ai/v3/openai';
+    this.params = params;
+
 
     let config = {
       baseURL: this.url
@@ -17,10 +20,15 @@ export class Novita {
 
 	async sendRequest(turns, systemMessage, stop_seq='***') {
       let messages = [{'role': 'system', 'content': systemMessage}].concat(turns);
+
+      
+      messages = strictFormat(messages);
+      
       const pack = {
           model: this.model_name || "meta-llama/llama-3.1-70b-instruct",
           messages,
           stop: [stop_seq],
+          ...(this.params || {})
       };
 
       let res = null;
@@ -41,6 +49,18 @@ export class Novita {
               res = 'My brain disconnected, try again.';
           }
       }
+      if (res.includes('<think>')) {
+        let start = res.indexOf('<think>');
+        let end = res.indexOf('</think>') + 8;
+        if (start != -1) {
+          if (end != -1) {
+            res = res.substring(0, start) + res.substring(end);
+          } else {
+            res = res.substring(0, start+7);
+          }
+        }
+        res = res.trim();
+      }
       return res;
   }
 

src/models/prompter.js

@@ -1,24 +1,24 @@
 import { readFileSync, mkdirSync, writeFileSync} from 'fs';
 import { Examples } from '../utils/examples.js';
-import { getCommandDocs } from './commands/index.js';
+import { getCommandDocs } from '../agent/commands/index.js';
+import { getSkillDocs } from '../agent/library/index.js';
 import { stringifyTurns } from '../utils/text.js';
-import { getCommand } from './commands/index.js';
+import { getCommand } from '../agent/commands/index.js';
 import settings from '../../settings.js';
 
-import { Gemini } from '../models/gemini.js';
+import { Gemini } from './gemini.js';
-import { GPT } from '../models/gpt.js';
+import { GPT } from './gpt.js';
-import { Claude } from '../models/claude.js';
+import { Claude } from './claude.js';
-import { Mistral } from '../models/mistral.js';
+import { Mistral } from './mistral.js';
-import { ReplicateAPI } from '../models/replicate.js';
+import { ReplicateAPI } from './replicate.js';
-import { Local } from '../models/local.js';
+import { Local } from './local.js';
-import { Novita } from '../models/novita.js';
+import { Novita } from './novita.js';
-import { GroqCloudAPI } from '../models/groq.js';
+import { GroqCloudAPI } from './groq.js';
-import { HuggingFace } from '../models/huggingface.js';
+import { HuggingFace } from './huggingface.js';
-import { Qwen } from "../models/qwen.js";
+import { Qwen } from "./qwen.js";
-import { Grok } from "../models/grok.js";
+import { Grok } from "./grok.js";
+import { DeepSeek } from './deepseek.js';
 import {SkillLibrary} from "./library/skill_library.js";
-import { DeepSeek } from '../models/deepseek.js';
-
 export class Prompter {
     constructor(agent, fp) {
         this.agent = agent;
@@ -34,7 +34,6 @@ export class Prompter {
         this.coding_examples = null;
         
         let name = this.profile.name;
-        let chat = this.profile.model;
         this.cooldown = this.profile.cooldown ? this.profile.cooldown : 0;
         this.last_prompt_time = 0;
         this.awaiting_coding = false;
@@ -43,68 +42,22 @@ export class Prompter {
         let max_tokens = null;
         if (this.profile.max_tokens)
             max_tokens = this.profile.max_tokens;
-        if (typeof chat === 'string' || chat instanceof String) {
-            chat = {model: chat};
-            if (chat.model.includes('gemini'))
-                chat.api = 'google';
-            else if (chat.model.includes('gpt') || chat.model.includes('o1'))
-                chat.api = 'openai';
-            else if (chat.model.includes('claude'))
-                chat.api = 'anthropic';
-            else if (chat.model.includes('huggingface/'))
-                chat.api = "huggingface";
-            else if (chat.model.includes('meta/') || chat.model.includes('replicate/'))
-                chat.api = 'replicate';
-            else if (chat.model.includes('mistralai/') || chat.model.includes("mistral/"))
-                chat.api = 'mistral';
-            else if (chat.model.includes("groq/") || chat.model.includes("groqcloud/"))
-                chat.api = 'groq';
-            else if (chat.model.includes('novita/'))
-                chat.api = 'novita';
-            else if (chat.model.includes('qwen'))
-                chat.api = 'qwen';
-            else if (chat.model.includes('grok'))
-                chat.api = 'xai';
-            else if (chat.model.includes('deepseek'))
-                chat.api = 'deepseek';
-            else
-                chat.api = 'ollama';
-        }
 
-        console.log('Using chat settings:', chat);
+        let chat_model_profile = this._selectAPI(this.profile.model);
+        this.chat_model = this._createModel(chat_model_profile);
 
-        if (chat.api === 'google')
+        if (this.profile.code_model) {
-            this.chat_model = new Gemini(chat.model, chat.url);
+            let code_model_profile = this._selectAPI(this.profile.code_model);
-        else if (chat.api === 'openai')
+            this.code_model = this._createModel(code_model_profile);
-            this.chat_model = new GPT(chat.model, chat.url);
+        }
-        else if (chat.api === 'anthropic')
+        else {
-            this.chat_model = new Claude(chat.model, chat.url);
+            this.code_model = this.chat_model;
-        else if (chat.api === 'replicate')
-            this.chat_model = new ReplicateAPI(chat.model, chat.url);
-        else if (chat.api === 'ollama')
-            this.chat_model = new Local(chat.model, chat.url);
-        else if (chat.api === 'mistral')
-            this.chat_model = new Mistral(chat.model, chat.url);
-        else if (chat.api === 'groq') {
-            this.chat_model = new GroqCloudAPI(chat.model.replace('groq/', '').replace('groqcloud/', ''), chat.url, max_tokens ? max_tokens : 8192);
         }
-        else if (chat.api === 'huggingface')
-            this.chat_model = new HuggingFace(chat.model, chat.url);
-        else if (chat.api === 'novita')
-            this.chat_model = new Novita(chat.model.replace('novita/', ''), chat.url);
-        else if (chat.api === 'qwen')
-            this.chat_model = new Qwen(chat.model, chat.url);
-        else if (chat.api === 'xai')
-            this.chat_model = new Grok(chat.model, chat.url);
-        else if (chat.api === 'deepseek')
-            this.chat_model = new DeepSeek(chat.model, chat.url);
-        else
-            throw new Error('Unknown API:', api);
 
         let embedding = this.profile.embedding;
         if (embedding === undefined) {
-            if (chat.api !== 'ollama')
+            if (chat_model_profile.api !== 'ollama')
-                embedding = {api: chat.api};
+                embedding = {api: chat_model_profile.api};
             else
                 embedding = {api: 'none'};
         }
@@ -146,6 +99,70 @@ export class Prompter {
         });
     }
 
+    _selectAPI(profile) {
+        if (typeof profile === 'string' || profile instanceof String) {
+            profile = {model: profile};
+        }
+        if (!profile.api) {
+            if (profile.model.includes('gemini'))
+                profile.api = 'google';
+            else if (profile.model.includes('gpt') || profile.model.includes('o1')|| profile.model.includes('o3'))
+                profile.api = 'openai';
+            else if (profile.model.includes('claude'))
+                profile.api = 'anthropic';
+            else if (profile.model.includes('huggingface/'))
+                profile.api = "huggingface";
+            else if (profile.model.includes('replicate/'))
+                profile.api = 'replicate';
+            else if (profile.model.includes('mistralai/') || profile.model.includes("mistral/"))
+                model_profile.api = 'mistral';
+            else if (profile.model.includes("groq/") || profile.model.includes("groqcloud/"))
+                profile.api = 'groq';
+            else if (profile.model.includes('novita/'))
+                profile.api = 'novita';
+            else if (profile.model.includes('qwen'))
+                profile.api = 'qwen';
+            else if (profile.model.includes('grok'))
+                profile.api = 'xai';
+            else if (profile.model.includes('deepseek'))
+                profile.api = 'deepseek';
+            else
+            profile.api = 'ollama';
+        }
+        return profile;
+    }
+
+    _createModel(profile) {
+        let model = null;
+        if (profile.api === 'google')
+            model = new Gemini(profile.model, profile.url, profile.params);
+        else if (profile.api === 'openai')
+            model = new GPT(profile.model, profile.url, profile.params);
+        else if (profile.api === 'anthropic')
+            model = new Claude(profile.model, profile.url, profile.params);
+        else if (profile.api === 'replicate')
+            model = new ReplicateAPI(profile.model, profile.url, profile.params);
+        else if (profile.api === 'ollama')
+            model = new Local(profile.model, profile.url, profile.params);
+        else if (profile.api === 'mistral')
+            model = new Mistral(profile.model, profile.url, profile.params);
+        else if (profile.api === 'groq')
+            model = new GroqCloudAPI(profile.model.replace('groq/', '').replace('groqcloud/', ''), profile.url, profile.params);
+        else if (profile.api === 'huggingface')
+            model = new HuggingFace(profile.model, profile.url, profile.params);
+        else if (profile.api === 'novita')
+            model = new Novita(profile.model.replace('novita/', ''), profile.url, profile.params);
+        else if (profile.api === 'qwen')
+            model = new Qwen(profile.model, profile.url, profile.params);
+        else if (profile.api === 'xai')
+            model = new Grok(profile.model, profile.url, profile.params);
+        else if (profile.api === 'deepseek')
+            model = new DeepSeek(profile.model, profile.url, profile.params);
+        else
+            throw new Error('Unknown API:', profile.api);
+        return model;
+    }
+
     getName() {
         return this.profile.name;
     }
@@ -285,7 +302,7 @@ export class Prompter {
         await this.checkCooldown();
         let prompt = this.profile.coding;
         prompt = await this.replaceStrings(prompt, messages, this.coding_examples);
-        let resp = await this.chat_model.sendRequest(messages, prompt);
+        let resp = await this.code_model.sendRequest(messages, prompt);
         this.awaiting_coding = false;
         return resp;
     }

src/models/qwen.js

@@ -1,81 +1,109 @@
-import OpenAIApi from 'openai';
+// This code uses Dashscope and HTTP to ensure the latest support for the Qwen model.
-import { getKey, hasKey } from '../utils/keys.js';
+// Qwen is also compatible with the OpenAI API format;
-import { strictFormat } from '../utils/text.js';
+
+import { getKey } from '../utils/keys.js';
 
 export class Qwen {
-    constructor(model_name, url) {
+    constructor(model_name, url, params) {
         this.model_name = model_name;
-
+        this.params = params;
-        let config = {};
+        this.url = url || 'https://dashscope.aliyuncs.com/api/v1/services/aigc/text-generation/generation';
-
+        this.apiKey = getKey('QWEN_API_KEY');
-        config.baseURL = url || 'https://dashscope.aliyuncs.com/compatible-mode/v1';
-        config.apiKey = getKey('QWEN_API_KEY');
-
-        this.openai = new OpenAIApi(config);
     }
 
-    async sendRequest(turns, systemMessage, stop_seq='***') {
+    async sendRequest(turns, systemMessage, stopSeq = '***', retryCount = 0) {
-        let messages = [{'role': 'system', 'content': systemMessage}].concat(turns);
+        if (retryCount > 5) {
+            console.error('Maximum retry attempts reached.');
+            return 'Error: Too many retry attempts.';
+        }
 
-        messages = strictFormat(messages);
+        const data = {
-
+            model: this.modelName || 'qwen-plus',
-        const pack = {
+            input: { messages: [{ role: 'system', content: systemMessage }, ...turns] },
-            model: this.model_name || "qwen-plus",
+            parameters: { 
-            messages,
+                result_format: 'message', 
-            stop: stop_seq,
+                stop: stopSeq,
+                ...(this.params || {})
+            },
         };
 
-        let res = null;
+        // Add default user message if all messages are 'system' role
+        if (turns.every((msg) => msg.role === 'system')) {
+            data.input.messages.push({ role: 'user', content: 'hello' });
+        }
+
+        if (!data.model || !data.input || !data.input.messages || !data.parameters) {
+            console.error('Invalid request data format:', data);
+            throw new Error('Invalid request data format.');
+        }
+
         try {
-            console.log('Awaiting Qwen api response...')
+            const response = await this._makeHttpRequest(this.url, data);
-            // console.log('Messages:', messages);
+            const choice = response?.output?.choices?.[0];
-            let completion = await this.openai.chat.completions.create(pack);
+
-            if (completion.choices[0].finish_reason == 'length')
+            if (choice?.finish_reason === 'length' && turns.length > 0) {
-                throw new Error('Context length exceeded');
+                return this.sendRequest(turns.slice(1), systemMessage, stopSeq, retryCount + 1);
-            console.log('Received.')
-            res = completion.choices[0].message.content;
-        }
-        catch (err) {
-            if ((err.message == 'Context length exceeded' || err.code == 'context_length_exceeded') && turns.length > 1) {
-                console.log('Context length exceeded, trying again with shorter context.');
-                return await this.sendRequest(turns.slice(1), systemMessage, stop_seq);
-            } else {
-                console.log(err);
-                res = 'My brain disconnected, try again.';
             }
+
+            return choice?.message?.content || 'No content received.';
+        } catch (err) {
+            console.error('Error occurred:', err);
+            return 'An error occurred, please try again.';
         }
-        return res;
     }
 
-    // Why random backoff?
-    // With a 30 requests/second limit on Alibaba Qwen's embedding service,
-    // random backoff helps maximize bandwidth utilization.
     async embed(text) {
-        const maxRetries = 5; // Maximum number of retries
+        if (!text || typeof text !== 'string') {
-        for (let retries = 0; retries < maxRetries; retries++) {
+            console.error('Invalid embedding input: text must be a non-empty string.');
-            try {
+            return 'Invalid embedding input: text must be a non-empty string.';
-                const { data } = await this.openai.embeddings.create({
+        }
-                    model: this.model_name || "text-embedding-v3",
+
-                    input: text,
+        const data = {
-                    encoding_format: "float",
+            model: 'text-embedding-v2',
-                });
+            input: { texts: [text] },
-                return data[0].embedding;
+            parameters: { text_type: 'query' },
-            } catch (err) {
+        };
-                if (err.status === 429) {
+
-                    // If a rate limit error occurs, calculate the exponential backoff with a random delay (1-5 seconds)
+        if (!data.model || !data.input || !data.input.texts || !data.parameters) {
-                    const delay = Math.pow(2, retries) * 1000 + Math.floor(Math.random() * 2000);
+            console.error('Invalid embedding request data format:', data);
-                    // console.log(`Rate limit hit, retrying in ${delay} ms...`);
+            throw new Error('Invalid embedding request data format.');
-                    await new Promise(resolve => setTimeout(resolve, delay)); // Wait for the delay before retrying
+        }
-                } else {
+
-                    throw err;
+        try {
-                }
+            const response = await this._makeHttpRequest(this.url, data);
-            }
+            const embedding = response?.output?.embeddings?.[0]?.embedding;
+            return embedding || 'No embedding result received.';
+        } catch (err) {
+            console.error('Error occurred:', err);
+            return 'An error occurred, please try again.';
         }
-        // If maximum retries are reached and the request still fails, throw an error
-        throw new Error('Max retries reached, request failed.');
     }
 
+    async _makeHttpRequest(url, data) {
+        const headers = {
+            'Authorization': `Bearer ${this.apiKey}`,
+            'Content-Type': 'application/json',
+        };
+
+        const response = await fetch(url, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify(data),
+        });
+
+        if (!response.ok) {
+            const errorText = await response.text();
+            console.error(`Request failed, status code ${response.status}: ${response.statusText}`);
+            console.error('Error response content:', errorText);
+            throw new Error(`Request failed, status code ${response.status}: ${response.statusText}`);
+        }
+
+        const responseText = await response.text();
+        try {
+            return JSON.parse(responseText);
+        } catch (err) {
+            console.error('Failed to parse response JSON:', err);
+            throw new Error('Invalid response JSON format.');
+        }
+    }
 }
-
-
-

src/models/replicate.js

@@ -4,9 +4,10 @@ import { getKey } from '../utils/keys.js';
 
 // llama, mistral
 export class ReplicateAPI {
-	constructor(model_name, url) {
+	constructor(model_name, url, params) {
 		this.model_name = model_name;
 		this.url = url;
+		this.params = params;
 
 		if (this.url) {
 			console.warn('Replicate API does not support custom URLs. Ignoring provided URL.');
@@ -22,7 +23,11 @@ export class ReplicateAPI {
 		const prompt = toSinglePrompt(turns, null, stop_seq);
 		let model_name = this.model_name || 'meta/meta-llama-3-70b-instruct';
 
-		const input = { prompt, system_prompt: systemMessage };
+		const input = { 
+			prompt, 
+			system_prompt: systemMessage,
+			...(this.params || {})
+		};
 		let res = null;
 		try {
 			console.log('Awaiting Replicate API response...');

src/process/init_agent.js

@@ -57,11 +57,8 @@ const argv = yargs(args)
         const agent = new Agent();
         await agent.start(argv.profile, argv.load_memory, argv.init_message, argv.count_id, argv.task_path, argv.task_id);
     } catch (error) {
-        console.error('Failed to start agent process:', {
+        console.error('Failed to start agent process:');
-            message: error.message || 'No error message',
+        console.error(error);
-            stack: error.stack || 'No stack trace',
-            error: error
-        });
         process.exit(1);
     }
 })();

src/utils/mcdata.js

@@ -190,7 +190,10 @@ export function getItemCraftingRecipes(itemName) {
                 recipe[ingredientName] = 0;
             recipe[ingredientName]++;
         }
-        recipes.push(recipe);
+        recipes.push([
+            recipe,
+            {craftedCount : r.result.count}
+        ]);
     }
 
     return recipes;
@@ -327,4 +330,156 @@ export function calculateLimitingResource(availableItems, requiredItems, discret
     }
     if(discrete) num = Math.floor(num);
     return {num, limitingResource}
+}
+
+let loopingItems = new Set();
+
+export function initializeLoopingItems() {
+
+    loopingItems = new Set(['coal',
+        'wheat',
+        'diamond',
+        'emerald',
+        'raw_iron',
+        'raw_gold',
+        'redstone',
+        'blue_wool',
+        'packed_mud',
+        'raw_copper',
+        'iron_ingot',
+        'dried_kelp',
+        'gold_ingot',
+        'slime_ball',
+        'black_wool',
+        'quartz_slab',
+        'copper_ingot',
+        'lapis_lazuli',
+        'honey_bottle',
+        'rib_armor_trim_smithing_template',
+        'eye_armor_trim_smithing_template',
+        'vex_armor_trim_smithing_template',
+        'dune_armor_trim_smithing_template',
+        'host_armor_trim_smithing_template',
+        'tide_armor_trim_smithing_template',
+        'wild_armor_trim_smithing_template',
+        'ward_armor_trim_smithing_template',
+        'coast_armor_trim_smithing_template',
+        'spire_armor_trim_smithing_template',
+        'snout_armor_trim_smithing_template',
+        'shaper_armor_trim_smithing_template',
+        'netherite_upgrade_smithing_template',
+        'raiser_armor_trim_smithing_template',
+        'sentry_armor_trim_smithing_template',
+        'silence_armor_trim_smithing_template',
+        'wayfinder_armor_trim_smithing_template']);
+}
+
+
+/**
+ * Gets a detailed plan for crafting an item considering current inventory
+ */
+export function getDetailedCraftingPlan(targetItem, count = 1, current_inventory = {}) {
+    initializeLoopingItems();
+    if (!targetItem || count <= 0 || !getItemId(targetItem)) {
+        return "Invalid input. Please provide a valid item name and positive count.";
+    }
+
+    if (isBaseItem(targetItem)) {
+        const available = current_inventory[targetItem] || 0;
+        if (available >= count) return "You have all required items already in your inventory!";
+        return `${targetItem} is a base item, you need to find ${count - available} more in the world`;
+    }
+
+    const inventory = { ...current_inventory };
+    const leftovers = {};
+    const plan = craftItem(targetItem, count, inventory, leftovers);
+    return formatPlan(plan);
+}
+
+function isBaseItem(item) {
+    return loopingItems.has(item) || getItemCraftingRecipes(item) === null;
+}
+
+function craftItem(item, count, inventory, leftovers, crafted = { required: {}, steps: [], leftovers: {} }) {
+    // Check available inventory and leftovers first
+    const availableInv = inventory[item] || 0;
+    const availableLeft = leftovers[item] || 0;
+    const totalAvailable = availableInv + availableLeft;
+
+    if (totalAvailable >= count) {
+        // Use leftovers first, then inventory
+        const useFromLeft = Math.min(availableLeft, count);
+        leftovers[item] = availableLeft - useFromLeft;
+        
+        const remainingNeeded = count - useFromLeft;
+        if (remainingNeeded > 0) {
+            inventory[item] = availableInv - remainingNeeded;
+        }
+        return crafted;
+    }
+
+    // Use whatever is available
+    const stillNeeded = count - totalAvailable;
+    if (availableLeft > 0) leftovers[item] = 0;
+    if (availableInv > 0) inventory[item] = 0;
+
+    if (isBaseItem(item)) {
+        crafted.required[item] = (crafted.required[item] || 0) + stillNeeded;
+        return crafted;
+    }
+
+    const recipe = getItemCraftingRecipes(item)?.[0];
+    if (!recipe) {
+        crafted.required[item] = stillNeeded;
+        return crafted;
+    }
+
+    const [ingredients, result] = recipe;
+    const craftedPerRecipe = result.craftedCount;
+    const batchCount = Math.ceil(stillNeeded / craftedPerRecipe);
+    const totalProduced = batchCount * craftedPerRecipe;
+
+    // Add excess to leftovers
+    if (totalProduced > stillNeeded) {
+        leftovers[item] = (leftovers[item] || 0) + (totalProduced - stillNeeded);
+    }
+
+    // Process each ingredient
+    for (const [ingredientName, ingredientCount] of Object.entries(ingredients)) {
+        const totalIngredientNeeded = ingredientCount * batchCount;
+        craftItem(ingredientName, totalIngredientNeeded, inventory, leftovers, crafted);
+    }
+
+    // Add crafting step
+    const stepIngredients = Object.entries(ingredients)
+        .map(([name, amount]) => `${amount * batchCount} ${name}`)
+        .join(' + ');
+    crafted.steps.push(`Craft ${stepIngredients} -> ${totalProduced} ${item}`);
+
+    return crafted;
+}
+
+function formatPlan({ required, steps, leftovers }) {
+    const lines = [];
+
+    if (Object.keys(required).length > 0) {
+        lines.push('You are missing the following items:');
+        Object.entries(required).forEach(([item, count]) => 
+            lines.push(`- ${count} ${item}`));
+        lines.push('\nOnce you have these items, here\'s your crafting plan:');
+    } else {
+        lines.push('You have all items required to craft this item!');
+        lines.push('Here\'s your crafting plan:');
+    }
+
+    lines.push('');
+    lines.push(...steps);
+
+    if (Object.keys(leftovers).length > 0) {
+        lines.push('\nYou will have leftover:');
+        Object.entries(leftovers).forEach(([item, count]) => 
+            lines.push(`- ${count} ${item}`));
+    }
+
+    return lines.join('\n');
 }

src/utils/text.js

@@ -26,8 +26,10 @@ export function toSinglePrompt(turns, system=null, stop_seq='***', model_nicknam
     return prompt;
 }
 
-// ensures stricter turn order for anthropic/llama models
+// ensures stricter turn order and roles:
-// combines repeated messages from the same role, separates repeat assistant messages with filler user messages
+// - system messages are treated as user messages and prefixed with SYSTEM:
+// - combines repeated messages from users
+// - separates repeat assistant messages with filler user messages
 export function strictFormat(turns) {
     let prev_role = null;
     let messages = [];

viewer.html

@@ -26,9 +26,9 @@
     </div>
     <script>
       function updateLayout() {
-        var width = window.innerWidth;
+        let width = window.innerWidth;
-        var height = window.innerHeight;
+        let height = window.innerHeight;
-        var iframes = document.querySelectorAll('.iframe-wrapper');
+        let iframes = document.querySelectorAll('.iframe-wrapper');
         if (width > height) {
           iframes.forEach(function(iframe) {
             iframe.style.width = '50%';
@@ -43,10 +43,10 @@
       }
       window.addEventListener('resize', updateLayout);
       window.addEventListener('load', updateLayout);
-      var iframes = document.querySelectorAll('.iframe-wrapper');
+      let iframes = document.querySelectorAll('.iframe-wrapper');
       iframes.forEach(function(iframe) {
-        var port = iframe.getAttribute('data-port');
+        let port = iframe.getAttribute('data-port');
-        var loaded = false;
+        let loaded = false;
         function checkServer() {
           fetch('http://localhost:' + port, { method: 'HEAD' })
             .then(function(response) {