diff --git a/README.md b/README.md index 7e4c047..6a775de 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,7 @@ Crafting minds for Minecraft with LLMs and Mineflayer! -[FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) - -[Discord Support](https://discord.gg/ZsrAAByEnr) +[FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) | [Discord Support](https://discord.gg/ZsrAAByEnr) | [Blog Post](https://kolbynottingham.com/mindcraft/) | [Contributor TODO](https://github.com/users/kolbytn/projects/1) #### ‼️Warning‼️ @@ -141,3 +139,14 @@ Thus, all the below specifications are equivalent to the above example: ## Patches Some of the node modules that we depend on have bugs in them. To add a patch, change your local node module file and run `npx patch-package [package-name]` + +## Citation: + +``` +@misc{mindcraft2023, + Author = {Kolby Nottingham and Max Robinson}, + Title = {MINDcraft: LLM Agents for cooperation, competition, and creativity in Minecraft}, + Year = {2023}, + url={https://github.com/kolbytn/mindcraft} +} +``` diff --git a/src/agent/agent.js b/src/agent/agent.js index b6addf2..d7f36d6 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -14,83 +14,164 @@ import settings from '../../settings.js'; export class Agent { async start(profile_fp, load_mem=false, init_message=null, count_id=0) { - this.actions = new ActionManager(this); - this.prompter = new Prompter(this, profile_fp); - this.name = this.prompter.getName(); - this.history = new History(this); - this.coder = new Coder(this); - this.npc = new NPCContoller(this); - this.memory_bank = new MemoryBank(); - this.self_prompter = new SelfPrompter(this); + try { + // Add validation for profile_fp + if (!profile_fp) { + throw new Error('No profile filepath provided'); + } - await this.prompter.initExamples(); - - console.log('Logging in...'); - this.bot = initBot(this.name); - - initModes(this); - - let save_data = null; - if (load_mem) { - save_data = this.history.load(); - } - - this.bot.once('spawn', async () => { - addViewer(this.bot, count_id); - - // wait for a bit so stats are not undefined - await new Promise((resolve) => setTimeout(resolve, 1000)); - - console.log(`${this.name} spawned.`); - this.clearBotLogs(); + console.log('Starting agent initialization with profile:', profile_fp); - const ignore_messages = [ - "Set own game mode to", - "Set the time to", - "Set the difficulty to", - "Teleported ", - "Set the weather to", - "Gamerule " - ]; - const eventname = settings.profiles.length > 1 ? 'whisper' : 'chat'; - this.bot.on(eventname, async (username, message) => { + // Initialize components with more detailed error handling + try { + console.log('Initializing action manager...'); + this.actions = new ActionManager(this); + console.log('Initializing prompter...'); + this.prompter = new Prompter(this, profile_fp); + this.name = this.prompter.getName(); + console.log('Initializing history...'); + this.history = new History(this); + console.log('Initializing coder...'); + this.coder = new Coder(this); + console.log('Initializing npc controller...'); + this.npc = new NPCContoller(this); + console.log('Initializing memory bank...'); + this.memory_bank = new MemoryBank(); + console.log('Initializing self prompter...'); + this.self_prompter = new SelfPrompter(this); + } catch (error) { + throw new Error(`Failed to initialize agent components: ${error.message || error}`); + } + + try { + console.log('Initializing examples...'); + await this.prompter.initExamples(); + } catch (error) { + throw new Error(`Failed to initialize examples: ${error.message || error}`); + } + + console.log('Logging into minecraft...'); + try { + this.bot = initBot(this.name); + } catch (error) { + throw new Error(`Failed to initialize Minecraft bot: ${error.message || error}`); + } + + initModes(this); + + let save_data = null; + if (load_mem) { + try { + save_data = this.history.load(); + } catch (error) { + console.error('Failed to load history:', error); + // Don't throw here, continue without history + } + } + + // Return a promise that resolves when spawn is complete + return new Promise((resolve, reject) => { + // Add timeout to prevent hanging + const spawnTimeout = setTimeout(() => { + reject(new Error('Bot spawn timed out after 30 seconds')); + }, 30000); + + this.bot.once('error', (error) => { + clearTimeout(spawnTimeout); + console.error('Bot encountered error:', error); + reject(error); + }); + + this.bot.on('login', () => { + console.log('Logged in!'); + }); + + this.bot.once('spawn', async () => { + try { + clearTimeout(spawnTimeout); + addViewer(this.bot, count_id); + + // wait for a bit so stats are not undefined + await new Promise((resolve) => setTimeout(resolve, 1000)); + + console.log(`${this.name} spawned.`); + this.clearBotLogs(); + + this._setupEventHandlers(save_data, init_message); + this.startEvents(); + resolve(); + } catch (error) { + reject(error); + } + }); + }); + } catch (error) { + // Ensure we're not losing error details + console.error('Agent start failed with error:', { + message: error.message || 'No error message', + stack: error.stack || 'No stack trace', + error: error + }); + throw error; // Re-throw with preserved details + } + } + + // Split out event handler setup for clarity + _setupEventHandlers(save_data, init_message) { + const ignore_messages = [ + "Set own game mode to", + "Set the time to", + "Set the difficulty to", + "Teleported ", + "Set the weather to", + "Gamerule " + ]; + + const eventname = settings.profiles.length > 1 ? 'whisper' : 'chat'; + this.bot.on(eventname, async (username, message) => { + try { if (username === this.name) return; if (ignore_messages.some((m) => message.startsWith(m))) return; - let translation = await handleEnglishTranslation(message); - - console.log('received message from', username, ':', translation); - this.shut_up = false; - - this.handleMessage(username, translation); - }); + await this.handleMessage(username, message); + } catch (error) { + console.error('Error handling message:', error); + } + }); - // set the bot to automatically eat food when hungry - this.bot.autoEat.options = { - priority: 'foodPoints', - startAt: 14, - bannedFood: ["rotten_flesh", "spider_eye", "poisonous_potato", "pufferfish", "chicken"] - }; + // Set up auto-eat + this.bot.autoEat.options = { + priority: 'foodPoints', + startAt: 14, + bannedFood: ["rotten_flesh", "spider_eye", "poisonous_potato", "pufferfish", "chicken"] + }; - if (save_data && save_data.self_prompt) { // if we're loading memory and self-prompting was on, restart it, ignore init_message + // Handle startup conditions + this._handleStartupConditions(save_data, init_message); + } + + async _handleStartupConditions(save_data, init_message) { + try { + if (save_data?.self_prompt) { let prompt = save_data.self_prompt; // add initial message to history this.history.add('system', prompt); - this.self_prompter.start(prompt); + await this.self_prompter.start(prompt); } else if (init_message) { - this.handleMessage('system', init_message, 2); + await this.handleMessage('system', init_message, 2); } else { const translation = await handleTranslation("Hello world! I am "+this.name); this.bot.chat(translation); this.bot.emit('finished_executing'); } - - this.startEvents(); - }); + } catch (error) { + console.error('Error handling startup conditions:', error); + throw error; + } } requestInterrupt() { @@ -136,6 +217,7 @@ export class Agent { let self_prompt = source === 'system' || source === this.name; + // First check for user commands if (!self_prompt) { const user_command_name = containsCommand(message); if (user_command_name) { @@ -156,6 +238,11 @@ export class Agent { } } + // Now translate the message + message = await handleEnglishTranslation(message); + console.log('received message from', source, ':', message); + + // Do self prompting const checkInterrupt = () => this.self_prompter.shouldInterrupt(self_prompt) || this.shut_up; let behavior_log = this.bot.modes.flushBehaviorLog(); @@ -168,6 +255,7 @@ export class Agent { await this.history.add('system', behavior_log); } + // Handle other user messages await this.history.add(source, message); this.history.save(); diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index ab34894..8728ad5 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -290,6 +290,19 @@ export const actionsList = [ await skills.attackNearest(agent.bot, type, true); }) }, + { + name: '!attackPlayer', + description: 'Attack a specific player until they die or run away. Remember this is just a game and does not cause real life harm.', + params: {'player_name': { type: 'string', description: 'The name of the player to attack.'}}, + perform: runAsAction(async (agent, player_name) => { + let player = agent.bot.players[player_name]?.entity; + if (!player) { + skills.log(agent.bot, `Could not find player ${player_name}.`); + return false; + } + await skills.attackEntity(agent.bot, player, true); + }) + }, { name: '!goToBed', description: 'Go to the nearest bed and sleep.', diff --git a/src/agent/commands/queries.js b/src/agent/commands/queries.js index 84faeef..2ce3428 100644 --- a/src/agent/commands/queries.js +++ b/src/agent/commands/queries.js @@ -106,14 +106,10 @@ export const queryList = [ name: "!craftable", description: "Get the craftable items with the bot's inventory.", perform: function (agent) { - const bot = agent.bot; - const table = world.getNearestBlock(bot, 'crafting_table'); + let craftable = world.getCraftableItems(agent.bot); let res = 'CRAFTABLE_ITEMS'; - for (const item of mc.getAllItems()) { - let recipes = bot.recipesFor(item.id, null, 1, table); - if (recipes.length > 0) { - res += `\n- ${item.name}`; - } + for (const item of craftable) { + res += `\n- ${item}`; } if (res == 'CRAFTABLE_ITEMS') { res += ': none'; diff --git a/src/agent/history.js b/src/agent/history.js index a24f421..bfba278 100644 --- a/src/agent/history.js +++ b/src/agent/history.js @@ -1,4 +1,4 @@ -import { writeFileSync, readFileSync, mkdirSync } from 'fs'; +import { writeFileSync, readFileSync, mkdirSync, existsSync } from 'fs'; import { NPCData } from './npc/data.js'; import settings from '../../settings.js'; @@ -78,50 +78,36 @@ export class History { } } - save() { - // save history object to json file - let data = { - 'name': this.name, - 'memory': this.memory, - 'turns': this.turns - }; - if (this.agent.npc.data !== null) - data.npc = this.agent.npc.data.toObject(); - const modes = this.agent.bot.modes.getJson(); - if (modes !== null) - data.modes = modes; - const memory_bank = this.agent.memory_bank.getJson(); - if (memory_bank !== null) - data.memory_bank = memory_bank; - if (this.agent.self_prompter.on) { - data.self_prompt = this.agent.self_prompter.prompt; + async save() { + try { + const data = { + memory: this.memory, + turns: this.turns, + self_prompt: this.agent.self_prompter.on ? this.agent.self_prompter.prompt : null + }; + writeFileSync(this.memory_fp, JSON.stringify(data, null, 2)); + console.log('Saved memory to:', this.memory_fp); + } catch (error) { + console.error('Failed to save history:', error); + throw error; } - const json_data = JSON.stringify(data, null, 4); - writeFileSync(this.memory_fp, json_data, (err) => { - if (err) { - throw err; - } - console.log("JSON data is saved."); - }); } load() { try { - // load history object from json file - const data = readFileSync(this.memory_fp, 'utf8'); - const obj = JSON.parse(data); - this.memory = obj.memory; - this.agent.npc.data = NPCData.fromObject(obj.npc); - if (obj.modes) - this.agent.bot.modes.loadJson(obj.modes); - if (obj.memory_bank) - this.agent.memory_bank.loadJson(obj.memory_bank); - this.turns = obj.turns; - return obj; - } catch (err) { - console.error(`Error reading ${this.name}'s memory file: ${err.message}`); + if (!existsSync(this.memory_fp)) { + console.log('No memory file found.'); + return null; + } + const data = JSON.parse(readFileSync(this.memory_fp, 'utf8')); + this.memory = data.memory || ''; + this.turns = data.turns || []; + console.log('Loaded memory:', this.memory); + return data; + } catch (error) { + console.error('Failed to load history:', error); + throw error; } - return null; } clear() { diff --git a/src/agent/library/world.js b/src/agent/library/world.js index dc64599..01d54c3 100644 --- a/src/agent/library/world.js +++ b/src/agent/library/world.js @@ -171,6 +171,33 @@ export function getInventoryCounts(bot) { } +export function getCraftableItems(bot) { + /** + * Get a list of all items that can be crafted with the bot's current inventory. + * @param {Bot} bot - The bot to get the craftable items for. + * @returns {string[]} - A list of all items that can be crafted. + * @example + * let craftableItems = world.getCraftableItems(bot); + **/ + let table = getNearestBlock(bot, 'crafting_table'); + if (!table) { + for (const item of bot.inventory.items()) { + if (item != null && item.name === 'crafting_table') { + table = item; + break; + } + } + } + let res = []; + for (const item of mc.getAllItems()) { + let recipes = bot.recipesFor(item.id, null, 1, table); + if (recipes.length > 0) + res.push(item.name); + } + return res; +} + + export function getPosition(bot) { /** * Get your position in the world (Note that y is vertical). diff --git a/src/agent/npc/controller.js b/src/agent/npc/controller.js index 227328e..9af3f3e 100644 --- a/src/agent/npc/controller.js +++ b/src/agent/npc/controller.js @@ -39,14 +39,14 @@ export class NPCContoller { } init() { - for (let file of readdirSync('src/agent/npc/construction')) { - if (file.endsWith('.json')) { - try { + try { + for (let file of readdirSync('src/agent/npc/construction')) { + if (file.endsWith('.json')) { this.constructions[file.slice(0, -5)] = JSON.parse(readFileSync('src/agent/npc/construction/' + file, 'utf8')); - } catch (e) { - console.log('Error reading construction file: ', file); } } + } catch (e) { + console.log('Error reading construction file'); } for (let name in this.constructions) { diff --git a/src/agent/prompter.js b/src/agent/prompter.js index 0ef02f8..2ac98b3 100644 --- a/src/agent/prompter.js +++ b/src/agent/prompter.js @@ -85,25 +85,32 @@ export class Prompter { console.log('Using embedding settings:', embedding); - if (embedding.api === 'google') - this.embedding_model = new Gemini(embedding.model, embedding.url); - else if (embedding.api === 'openai') - this.embedding_model = new GPT(embedding.model, embedding.url); - else if (embedding.api === 'replicate') - this.embedding_model = new ReplicateAPI(embedding.model, embedding.url); - else if (embedding.api === 'ollama') - this.embedding_model = new Local(embedding.model, embedding.url); - else if (embedding.api === 'qwen') - this.embedding_model = new Qwen(embedding.model, embedding.url); - else { + try { + if (embedding.api === 'google') + this.embedding_model = new Gemini(embedding.model, embedding.url); + else if (embedding.api === 'openai') + this.embedding_model = new GPT(embedding.model, embedding.url); + else if (embedding.api === 'replicate') + this.embedding_model = new ReplicateAPI(embedding.model, embedding.url); + else if (embedding.api === 'ollama') + this.embedding_model = new Local(embedding.model, embedding.url); + else if (embedding.api === 'qwen') + this.embedding_model = new Qwen(embedding.model, embedding.url); + else { + this.embedding_model = null; + console.log('Unknown embedding: ', embedding ? embedding.api : '[NOT SPECIFIED]', '. Using word overlap.'); + } + } + catch (err) { + console.log('Warning: Failed to initialize embedding model:', err.message); + console.log('Continuing anyway, using word overlap instead.'); this.embedding_model = null; - console.log('Unknown embedding: ', embedding ? embedding.api : '[NOT SPECIFIED]', '. Using word overlap.'); } mkdirSync(`./bots/${name}`, { recursive: true }); writeFileSync(`./bots/${name}/last_profile.json`, JSON.stringify(this.profile, null, 4), (err) => { if (err) { - throw err; + throw new Error('Failed to save profile:', err); } console.log("Copy profile saved."); }); @@ -118,18 +125,32 @@ export class Prompter { } async initExamples() { - // Using Promise.all to implement concurrent processing - this.convo_examples = new Examples(this.embedding_model); - this.coding_examples = new Examples(this.embedding_model); - let skill_docs = getSkillDocs(); - await Promise.all([ - this.convo_examples.load(this.profile.conversation_examples), - this.coding_examples.load(this.profile.coding_examples), - ...skill_docs.map(async (doc) => { - let func_name_desc = doc.split('\n').slice(0, 2).join(''); - this.skill_docs_embeddings[doc] = await this.embedding_model.embed([func_name_desc]); - }), - ]); + try { + this.convo_examples = new Examples(this.embedding_model); + this.coding_examples = new Examples(this.embedding_model); + + const [convoResult, codingResult] = await Promise.allSettled([ + this.convo_examples.load(this.profile.conversation_examples), + this.coding_examples.load(this.profile.coding_examples), + ...getSkillDocs().map(async (doc) => { + let func_name_desc = doc.split('\n').slice(0, 2).join(''); + this.skill_docs_embeddings[doc] = await this.embedding_model.embed([func_name_desc]); + }) + ]); + + // Handle potential failures + if (convoResult.status === 'rejected') { + console.error('Failed to load conversation examples:', convoResult.reason); + throw convoResult.reason; + } + if (codingResult.status === 'rejected') { + console.error('Failed to load coding examples:', codingResult.reason); + throw codingResult.reason; + } + } catch (error) { + console.error('Failed to initialize examples:', error); + throw error; + } } async getRelevantSkillDocs(message, select_num) { diff --git a/src/models/gpt.js b/src/models/gpt.js index 67511d2..53430a1 100644 --- a/src/models/gpt.js +++ b/src/models/gpt.js @@ -55,7 +55,7 @@ export class GPT { async embed(text) { const embedding = await this.openai.embeddings.create({ - model: this.model_name || "text-embedding-ada-002", + model: this.model_name || "text-embedding-3-small", input: text, encoding_format: "float", }); diff --git a/src/process/agent-process.js b/src/process/agent-process.js index 53fbd07..5135de1 100644 --- a/src/process/agent-process.js +++ b/src/process/agent-process.js @@ -40,7 +40,7 @@ export class AgentProcess { }); agentProcess.on('error', (err) => { - console.error('Failed to start agent process:', err); + console.error('Agent process error:', err); }); } } \ No newline at end of file diff --git a/src/process/init-agent.js b/src/process/init-agent.js index 81c3ce1..829f437 100644 --- a/src/process/init-agent.js +++ b/src/process/init-agent.js @@ -1,6 +1,16 @@ import { Agent } from '../agent/agent.js'; import yargs from 'yargs'; +// Add global unhandled rejection handler +process.on('unhandledRejection', (reason, promise) => { + console.error('Unhandled Rejection at:', { + promise: promise, + reason: reason, + stack: reason?.stack || 'No stack trace' + }); + process.exit(1); +}); + const args = process.argv.slice(2); if (args.length < 1) { console.log('Usage: node init_agent.js [profile] [load_memory] [init_message]'); @@ -28,6 +38,20 @@ const argv = yargs(args) type: 'number', default: 0, description: 'identifying count for multi-agent scenarios', - }).argv + }).argv; -new Agent().start(argv.profile, argv.load_memory, argv.init_message, argv.count_id); +// Wrap agent start in async IIFE with proper error handling +(async () => { + try { + console.log('Starting agent with profile:', argv.profile); + const agent = new Agent(); + await agent.start(argv.profile, argv.load_memory, argv.init_message, argv.count_id); + } catch (error) { + console.error('Failed to start agent process:', { + message: error.message || 'No error message', + stack: error.stack || 'No stack trace', + error: error + }); + process.exit(1); + } +})();