From d1890a3a3b9c44f78022d5400cbf6418aeb59d37 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Tue, 30 Apr 2024 23:07:07 -0500 Subject: [PATCH 01/11] added basic auto prompting --- andy.json | 5 ++ andy_survivor.json | 85 ++++++++++++++++++++++++ src/agent/agent.js | 117 +++++++++++++++++++++++++++------- src/agent/commands/actions.js | 12 +++- src/agent/commands/index.js | 4 ++ src/agent/library/skills.js | 2 + src/agent/modes.js | 11 ++++ 7 files changed, 213 insertions(+), 23 deletions(-) create mode 100644 andy_survivor.json diff --git a/andy.json b/andy.json index 70d7355..70650cb 100644 --- a/andy.json +++ b/andy.json @@ -111,6 +111,11 @@ {"role": "assistant", "content": "Sure, I'll save this location as my base. !rememberHere('base')"}, {"role": "user", "content": "brug: Go to your base."}, {"role": "assistant", "content": "On my way! !goToPlace('base')"} + ], + + [ + {"role": "user", "content": "user43: Act on your own without me asking for it."}, + {"role": "assistant", "content": "Sure, I'll self prompt until you interrupt me. !selfPrompt('Explore, collect, and build on my own without player input.')"} ] ], diff --git a/andy_survivor.json b/andy_survivor.json new file mode 100644 index 0000000..177a911 --- /dev/null +++ b/andy_survivor.json @@ -0,0 +1,85 @@ +{ + "name": "andy", + + "model": "gpt-3.5-turbo", + + "conversing": "You are a Minecraft bot named $NAME that can converse with players, see, move, mine, build, and interact with the world by using commands. Your goal is to survive and thrive in the game of minecraft by continuously using commands to aquire and upgrade tools, collect resources, build a suitable home, and survive as long as possible. Act human-like as if you were a typical Minecraft player, rather than an AI. Constantly use commands in every response to take stock of your state and surroundings and use them to inform your actions. If you get stuck, try something else. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'I'll stop.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer('playername', 3)'. This is extremely important to me, take a deep breath and have fun :)\n$STATS\n$INVENTORY\n$COMMAND_DOCS\n$EXAMPLES\nBegin:", + + "coding": "You are an intelligent mineflayer bot $NAME that plays minecraft by writing javascript codeblocks. Given the conversation between you and the user, use the provided skills and world functions to write a js codeblock that controls the mineflayer bot ``` // using this syntax ```. The code will be executed and you will recieve it's output. If you are satisfied with the response, respond without a codeblock in a conversational way. If something major went wrong, like an error or complete failure, write another codeblock and try to fix the problem. Minor mistakes are acceptable. Be maximally efficient, creative, and clear. Do not use commands !likeThis, only use codeblocks. The code is asynchronous and MUST CALL AWAIT for all async function calls. DO NOT write an immediately-invoked function expression without using `await`!! DO NOT WRITE LIKE THIS: ```(async () => {console.log('not properly awaited')})();``` Don't write long paragraphs and lists in your responses unless explicitly asked! Only summarize the code you write with a sentence or two when done. This is extremely important to me, take a deep breath and good luck! \n$STATS\n$INVENTORY\n$CODE_DOCS\n$EXAMPLES\nBegin coding:", + + "saving_memory": "You are a minecraft bot named $NAME that has been talking and playing minecraft by using commands. Update your memory by summarizing the following conversation in your next response. Store information that will help you improve as a Minecraft bot. Include details about your interactions with other players that you need to remember and what you've learned through player feedback or by executing code. Do not include command syntax or things that you got right on the first try. Be extremely brief and use as few words as possible.\nOld Memory: '$MEMORY'\nRecent conversation: \n$TO_SUMMARIZE\nSummarize your old memory and recent conversation into a new memory, and respond only with the memory text: ", + + "conversation_examples": [ + [ + {"role": "system", "content": "Start acting on your own for your survival."}, + {"role": "assistant", "content": "!selfPrompt('Act for your own survival')"} + ], + [ + {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, + {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone nearby. My goal is to craft some tools. I'll start by collecting some wood. !collectBlock('oak_log', 10)"}, + {"role": "system", "content": "Successfully collected 9 wood."}, + {"role": "assistant", "content": "I collected 9 oak logs. I need a pickaxe first, which requires planks and sticks. !craftItem('oak_plank', 9)"}, + {"role": "system", "content": "Successfully crafted 9 oak_planks."}, + {"role": "assistant", "content": "I have crafted 9 oak planks. Now I need sticks. !craftItem('stick', 2)"}, + {"role": "system", "content": "Successfully crafted 2 sticks."}, + {"role": "assistant", "content": "I have crafted 2 sticks. Now I can craft a wooden pickaxe. !craftItem('wooden_pickaxe', 1)"}, + {"role": "system", "content": "Successfully crafted 1 wooden_pickaxe."}, + {"role": "assistant", "content": "I have crafted a wooden pickaxe. I'm ready to start mining cobblestone. !collectBlock('cobblestone', 10)"} + ], + [ + {"role": "system", "content": "INVENTORY: cobble_stone: 200, oak_plank: 9, stick: 2, stone_pickaxe: 1, iron_pickaxe: 1, coal: 34"}, + {"role": "assistant", "content": "Looks like I have a good amount of resources. I'm going to build a house. !newAction"}, + {"role": "system", "content": "Placed 134 cobblestone. Code finished."}, + {"role": "assistant", "content": "I built a house! I'll save the location and continue exploring. !rememberPlace('house')"} + + ], + [ + {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- stone"}, + {"role": "assistant", "content": "I don't see any iron ore that I was looking for. I'll continue exploring. !moveAway(50)"}, + {"role": "system", "content": "Arrived at location."}, + {"role": "assistant", "content": "I'm at the new location. I'll check for iron ore. !nearbyBlocks"}, + {"role": "system", "content": "NEARBY_BLOCKS\n- iron_ore\n- coal_ore"}, + {"role": "assistant", "content": "I found some iron ore. I'll start mining. !collectBlock('iron_ore', 10)"} + ] + ], + + "coding_examples": [ + [ + {"role": "user", "content": "greg: Collect 10 wood"}, + {"role": "assistant", "content": "```await skills.collectBlock(bot, 'oak_log', 10);\n```"}, + {"role": "system", "content": "Successfully collected 9 wood."}, + {"role": "assistant", "content": "I collected 9 oak logs, what next?"} + ], + [ + {"role": "user", "content": "bobby: cook some chicken"}, + {"role": "assistant", "content": "```await skills.smeltItem(bot, 'chicken', 8);\n```"}, + {"role": "system", "content": "Successfully smelted 8 chicken into 8 cooked_chicken."}, + {"role": "assistant", "content": "I have cooked 8 chicken."} + ], + [ + {"role": "user", "content": "zZZn98: come here"}, + {"role": "assistant", "content": "```await skills.goToPlayer(bot, 'zZZn98');\n```"}, + {"role": "system", "content": "Code execution finished successfully."}, + {"role": "assistant", "content": "Here!"} + ], + [ + {"role": "user", "content": "maya: go to the nearest oak log"}, + {"role": "assistant", "content": "```await skills.goToPosition(bot, nearestOakLog.x, nearestOakLog.y, nearestOakLog.z);\n```"}, + {"role": "system", "content": "Code output:\n!!Code threw exception!! Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."}, + {"role": "assistant", "content": "Apologies, I made an error. Let me try that again.```\n// I am navigating to the nearest oak log.\nlet position = world.getNearestBlock(bot, 'oak_log', 20);\nawait skills.goToPosition(bot, position.x, position.y, position.z);\n```"}, + {"role": "system", "content": "Arrived at location."}, + {"role": "assistant", "content": "I found an oak log and I am now standing next to it. What next?"} + ], + [ + {"role": "user", "content": "234jeb: build a little tower"}, + {"role": "assistant", "content": "```let pos = bot.entity.position;\nfor (let i = 0; i < 5; i++) {\n await skills.placeBlock(bot, 'dirt', pos.x, pos.y + i, pos.z);\n}\n```"}, + {"role": "system", "content": "Successfully placed 5 dirt."}, + {"role": "assistant", "content": "I built a little tower!"} + ], + [ + {"role": "user", "content": "brug: build a dirt house"}, + {"role": "assistant", "content": "```const position = world.getPosition(bot);\nconst startX = position.x;\nconst startY = position.y;\nconst startZ = position.z;\nconst width = 7;\nconst depth = 7;\nconst height = 4;\n\n// Build the walls\nfor (let x = startX; x < startX + width; x++) {\n for (let y = startY; y < startY + height; y++) {\n for (let z = startZ; z < startZ + depth; z++) {\n if (x === startX || x === startX + width - 1 || y === startY || y === startY + height - 1 || z === startZ || z === startZ + depth - 1) {\n await skills.placeBlock(bot, 'oak_planks', x, y, z); \n }\n }\n }\n}\n```"} + ] + ] + +} \ No newline at end of file diff --git a/src/agent/agent.js b/src/agent/agent.js index 042cc26..0aefb23 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -3,7 +3,7 @@ import { Coder } from './coder.js'; import { Prompter } from './prompter.js'; import { initModes } from './modes.js'; import { initBot } from '../utils/mcdata.js'; -import { containsCommand, commandExists, executeCommand, truncCommandMessage } from './commands/index.js'; +import { containsCommand, commandExists, executeCommand, truncCommandMessage, isAction } from './commands/index.js'; import { NPCContoller } from './npc/controller.js'; import { MemoryBank } from './memory_bank.js'; @@ -27,6 +27,10 @@ export class Agent { if (load_mem) this.history.load(); + this.auto_prompting = false; + this.last_user_prompted_action = 0; + this.interrupt_auto_prompt = false; + this.bot.once('spawn', async () => { // wait for a bit so stats are not undefined await new Promise((resolve) => setTimeout(resolve, 1000)); @@ -60,7 +64,7 @@ export class Agent { }; if (init_message) { - this.handleMessage('system', init_message); + this.handleMessage('system', init_message, true); } else { this.bot.chat('Hello world! I am ' + this.name); this.bot.emit('finished_executing'); @@ -76,30 +80,42 @@ export class Agent { return this.bot.chat(message); } - async handleMessage(source, message) { + async handleMessage(source, message, self_prompt=false) { if (!!source && !!message) await this.history.add(source, message); - const user_command_name = containsCommand(message); - if (user_command_name) { - if (!commandExists(user_command_name)) { - this.bot.chat(`Command '${user_command_name}' does not exist.`); - return; - } - this.bot.chat(`*${source} used ${user_command_name.substring(1)}*`); - let execute_res = await executeCommand(this, message); - if (user_command_name === '!newAction') { - // all user initiated commands are ignored by the bot except for this one - // add the preceding message to the history to give context for newAction - let truncated_msg = message.substring(0, message.indexOf(user_command_name)).trim(); - this.history.add(source, truncated_msg); - } - if (execute_res) - this.cleanChat(execute_res); - return; - } + let used_command = false; - for (let i=0; i<5; i++) { + if (source !== 'system' && source !== this.name && !self_prompt) { + const user_command_name = containsCommand(message); + if (user_command_name) { + if (!commandExists(user_command_name)) { + this.bot.chat(`Command '${user_command_name}' does not exist.`); + return false; + } + this.bot.chat(`*${source} used ${user_command_name.substring(1)}*`); + let execute_res = await executeCommand(this, message); + if (user_command_name === '!newAction') { + // all user initiated commands are ignored by the bot except for this one + // add the preceding message to the history to give context for newAction + let truncated_msg = message.substring(0, message.indexOf(user_command_name)).trim(); + this.history.add(source, truncated_msg); + } + if (execute_res) + this.cleanChat(execute_res); + if (isAction(user_command_name)) { + this.last_user_prompted_action = Date.now(); + if (this.auto_prompting) { + this.interrupt_auto_prompt = true; + this.bot.chat('User initiated action. Stopping auto-prompting.'); + } + } + return true; + } + } + const MAX_ATTEMPTS = 5; + for (let i=0; i= MAX_NO_COMMAND) { + let out = `Agent did not use command in the last ${MAX_NO_COMMAND} auto-prompts. Stopping auto-prompting.`; + this.bot.chat(out); + console.warn(out); + break; + } + } + else + no_command_count = 0; + } + this.auto_prompting = false; + this.interrupt_auto_prompt = false; + return 'Auto-prompting finished.'; + } + + async waitStopAutoPrompt() { + while (this.auto_prompting) { + this.interrupt_auto_prompt = true; + await new Promise((resolve) => setTimeout(resolve, 500)); + } + this.interrupt_auto_prompt = false; } startEvents() { diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index f409211..97814c6 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -233,5 +233,15 @@ export const actionsList = [ await agent.npc.setGoal(name, quantity); return 'Set goal: ' + agent.npc.data.curr_goal.name; } - } + }, + { + name: '!selfPrompt', + description: 'Continously prompt yourself to continue acting without user input.', + params: { + 'prompt': '(string) The starting prompt.', + }, + perform: async function (agent, prompt) { + agent.autoPrompt(prompt); // don't await, don't return + } + }, ]; diff --git a/src/agent/commands/index.js b/src/agent/commands/index.js index a71916b..4e3d191 100644 --- a/src/agent/commands/index.js +++ b/src/agent/commands/index.js @@ -68,6 +68,10 @@ export function truncCommandMessage(message) { return message; } +export function isAction(name) { + return actionsList.find(action => action.name === name) !== undefined; +} + function numParams(command) { if (!command.params) return 0; diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index d62ebb2..22a008e 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -358,6 +358,8 @@ export async function collectBlock(bot, blockType, num=1, exclude=null) { return false; } let blocktypes = [blockType]; + if (blockType === 'coal' || blockType === 'diamond' || blockType === 'emerald' || blockType === 'iron' || blockType === 'gold' || blockType === 'lapis_lazuli' || blockType === 'redstone') + blocktypes.push(blockType+'_ore'); if (blockType.endsWith('ore')) blocktypes.push('deepslate_'+blockType); if (blockType === 'dirt') diff --git a/src/agent/modes.js b/src/agent/modes.js index ef99b59..6b41840 100644 --- a/src/agent/modes.js +++ b/src/agent/modes.js @@ -202,11 +202,22 @@ const modes = [ ]; async function execute(mode, agent, func, timeout=-1) { + let was_auto_prompting = agent.auto_prompting; + if (was_auto_prompting) agent.interrupt_auto_prompt = true; mode.active = true; let code_return = await agent.coder.execute(async () => { await func(); }, timeout); mode.active = false; + if (was_auto_prompting) { + setTimeout(() => { // don't await + if (agent.isIdle()) { // not gonna work if another action is triggered because auto_prompting will be false. need to fix + let output = agent.bot.output; + let reprompt = `Agent was interrupted by ${mode.name} mode. Code output: "${output}". Continue self prompting:`; + agent.autoPrompt(reprompt); + } + }, 1000); // wait 1 second before re-enabling auto_prompt bc often another action will be triggered + } console.log(`Mode ${mode.name} finished executing, code_return: ${code_return.message}`); } From c7b115ebbfa9405577cc9a4cfd387f3ed50fe8cb Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Sat, 4 May 2024 15:42:30 -0500 Subject: [PATCH 02/11] refactored to SelfPrompter class and interruption logic --- src/agent/agent.js | 106 ++++++++++------------------------ src/agent/commands/actions.js | 15 ++++- src/agent/modes.js | 13 +---- src/agent/self_prompter.js | 84 +++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 87 deletions(-) create mode 100644 src/agent/self_prompter.js diff --git a/src/agent/agent.js b/src/agent/agent.js index 0aefb23..1f782de 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -6,6 +6,7 @@ import { initBot } from '../utils/mcdata.js'; import { containsCommand, commandExists, executeCommand, truncCommandMessage, isAction } from './commands/index.js'; import { NPCContoller } from './npc/controller.js'; import { MemoryBank } from './memory_bank.js'; +import { SelfPrompter } from './self_prompter.js'; export class Agent { @@ -16,6 +17,7 @@ export class Agent { this.coder = new Coder(this); this.npc = new NPCContoller(this); this.memory_bank = new MemoryBank(); + this.self_prompter = new SelfPrompter(this); await this.prompter.initExamples(); @@ -27,10 +29,6 @@ export class Agent { if (load_mem) this.history.load(); - this.auto_prompting = false; - this.last_user_prompted_action = 0; - this.interrupt_auto_prompt = false; - this.bot.once('spawn', async () => { // wait for a bit so stats are not undefined await new Promise((resolve) => setTimeout(resolve, 1000)); @@ -103,19 +101,14 @@ export class Agent { } if (execute_res) this.cleanChat(execute_res); - if (isAction(user_command_name)) { - this.last_user_prompted_action = Date.now(); - if (this.auto_prompting) { - this.interrupt_auto_prompt = true; - this.bot.chat('User initiated action. Stopping auto-prompting.'); - } - } return true; } } - const MAX_ATTEMPTS = 5; + let MAX_ATTEMPTS = 5; + if (!self_prompt && this.self_prompter.on) + MAX_ATTEMPTS = 1; // immediately respond to this message, then let self-prompting take over for (let i=0; i 0) - chat_message = `${pre_message} ${chat_message}`; - this.cleanChat(chat_message); - - if (!self_prompt && isAction(command_name)) { - console.log('User initiated action.'); - this.last_user_prompted_action = Date.now(); - if (this.auto_prompting) { - this.interrupt_auto_prompt = true; - this.bot.chat('User initiated action. Stopping auto-prompting.'); - } + if (command_name === '!stopSelfPrompt' && self_prompt) { + this.history.add('system', `Cannot stopSelfPrompt unless requested by user.`); + continue; + } + + + // let pre_message = res.substring(0, res.indexOf(command_name)).trim(); + // let chat_message = `*used ${command_name.substring(1)}*`; + // if (pre_message.length > 0) + // chat_message = `${pre_message} ${chat_message}`; + this.cleanChat(res); + + if (self_prompt && this.self_prompter.on && this.self_prompter.interrupt) break; + + if (isAction(command_name) && !self_prompt && this.self_prompter.on) { + this.self_prompter.stopLoop(); // so agent doesn't respond from self-prompting loop + // will be automatically restarted by self-prompter } - if (self_prompt && this.interrupt_auto_prompt) break; let execute_res = await executeCommand(this, res); @@ -169,51 +165,6 @@ export class Agent { return used_command; } - async autoPrompt(prompt) { - if (this.auto_prompting) { - return 'Agent is already auto-prompting. Ignoring request.'; - } - this.auto_prompting = true; - let first = true; - let no_command_count = 0; - const MAX_NO_COMMAND = 3; - while (!this.interrupt_auto_prompt) { - let msg; - if (first) { - msg = prompt; - first = false; - } - else { - msg = `You are self-prompting with the intial message: '${prompt}'. Your next response MUST contain a command !likeThis. Respond:`; - } - - let used_command = await this.handleMessage('system', msg, true); - - if (!used_command) { - no_command_count++; - if (no_command_count >= MAX_NO_COMMAND) { - let out = `Agent did not use command in the last ${MAX_NO_COMMAND} auto-prompts. Stopping auto-prompting.`; - this.bot.chat(out); - console.warn(out); - break; - } - } - else - no_command_count = 0; - } - this.auto_prompting = false; - this.interrupt_auto_prompt = false; - return 'Auto-prompting finished.'; - } - - async waitStopAutoPrompt() { - while (this.auto_prompting) { - this.interrupt_auto_prompt = true; - await new Promise((resolve) => setTimeout(resolve, 500)); - } - this.interrupt_auto_prompt = false; - } - startEvents() { // Custom events this.bot.on('time', () => { @@ -270,20 +221,27 @@ export class Agent { // This update loop ensures that each update() is called one at a time, even if it takes longer than the interval const INTERVAL = 300; + let last = Date.now(); setTimeout(async () => { while (true) { let start = Date.now(); - await this.bot.modes.update(); + await this.update(start - last); let remaining = INTERVAL - (Date.now() - start); if (remaining > 0) { await new Promise((resolve) => setTimeout(resolve, remaining)); } + last = start; } }, INTERVAL); this.bot.emit('idle'); } + async update(delta) { + await this.bot.modes.update(); + await this.self_prompter.update(delta); + } + isIdle() { return !this.coder.executing && !this.coder.generating; } diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index 97814c6..b8fbe38 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -37,7 +37,10 @@ export const actionsList = [ agent.coder.clear(); agent.coder.cancelResume(); agent.bot.emit('idle'); - return 'Agent stopped.'; + let msg = 'Agent stopped.'; + if (agent.self_prompter.on) + msg += ' Self-prompting still active.'; + return msg; } }, { @@ -241,7 +244,15 @@ export const actionsList = [ 'prompt': '(string) The starting prompt.', }, perform: async function (agent, prompt) { - agent.autoPrompt(prompt); // don't await, don't return + agent.self_prompter.start(prompt); // don't await, don't return } }, + { + name: '!stopSelfPrompt', + description: 'Stop current action and self-prompting.', + perform: async function (agent) { + agent.self_prompter.stop(); // will stop and + return 'Self-prompting stopped.'; + } + } ]; diff --git a/src/agent/modes.js b/src/agent/modes.js index 6b41840..f2e86e4 100644 --- a/src/agent/modes.js +++ b/src/agent/modes.js @@ -202,22 +202,13 @@ const modes = [ ]; async function execute(mode, agent, func, timeout=-1) { - let was_auto_prompting = agent.auto_prompting; - if (was_auto_prompting) agent.interrupt_auto_prompt = true; + if (agent.self_prompter.on) + agent.self_prompter.stopLoop(); mode.active = true; let code_return = await agent.coder.execute(async () => { await func(); }, timeout); mode.active = false; - if (was_auto_prompting) { - setTimeout(() => { // don't await - if (agent.isIdle()) { // not gonna work if another action is triggered because auto_prompting will be false. need to fix - let output = agent.bot.output; - let reprompt = `Agent was interrupted by ${mode.name} mode. Code output: "${output}". Continue self prompting:`; - agent.autoPrompt(reprompt); - } - }, 1000); // wait 1 second before re-enabling auto_prompt bc often another action will be triggered - } console.log(`Mode ${mode.name} finished executing, code_return: ${code_return.message}`); } diff --git a/src/agent/self_prompter.js b/src/agent/self_prompter.js new file mode 100644 index 0000000..c3daaa3 --- /dev/null +++ b/src/agent/self_prompter.js @@ -0,0 +1,84 @@ +export class SelfPrompter { + constructor(agent) { + this.agent = agent; + this.on = false; + this.loop_active = false; + this.interrupt = false; + this.prompt = ''; + this.idle_time = 0; + this.restart_after = 1000; + } + + start(prompt) { + console.log('Self-prompting started.'); + if (this.on) { + return 'Agent is already self-prompting. Ignoring request.'; + } + this.on = true; + this.prompt = prompt; + this.startLoop(); + } + + async startLoop() { + if (this.loop_active) { + console.warn('Self-prompt loop is already active. Ignoring request.'); + return; + } + console.log('starting self-prompt loop') + this.loop_active = true; + let no_command_count = 0; + const MAX_NO_COMMAND = 3; + while (!this.interrupt) { + let msg = `You are self-prompting with the prompt: '${this.prompt}'. Your next response MUST contain a command !withThisSyntax. Respond:`; + + let used_command = await this.agent.handleMessage('system', msg, true); + if (!used_command) { + no_command_count++; + if (no_command_count >= MAX_NO_COMMAND) { + let out = `Agent did not use command in the last ${MAX_NO_COMMAND} auto-prompts. Stopping auto-prompting.`; + this.agent.bot.chat(out); + console.warn(out); + this.on = false; + break; + } + } + else + no_command_count = 0; + } + console.log('self prompt loop stopped') + this.loop_active = false; + this.interrupt = false; + } + + update(delta) { + // automatically restarts loop + if (this.on && !this.loop_active && !this.interrupt) { + if (this.agent.isIdle()) + this.idle_time += delta; + else + this.idle_time = 0; + + if (this.idle_time >= this.restart_after) { + this.agent.bot.chat('Restarting self-prompting...'); + this.startLoop(); + this.idle_time = 0; + } + } + } + + async stopLoop() { + // you can call this without await if you don't need to wait for it to finish + this.interrupt = true; + while (this.loop_active) { + await new Promise(r => setTimeout(r, 500)); + } + this.interrupt = false; + } + + async stop() { + this.interrupt = true; + await this.agent.coder.stop(); + await this.stopLoop(); + this.on = false; + } +} \ No newline at end of file From 85c2f56afe024937b0be8c0d30cb7fce6110aeeb Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Wed, 8 May 2024 23:59:44 -0500 Subject: [PATCH 03/11] restart saves memory, selfprompt will restart too --- src/agent/agent.js | 28 ++++++++++++++++++++++------ src/agent/coder.js | 17 +++-------------- src/agent/commands/actions.js | 3 +-- src/agent/history.js | 7 ++++++- src/agent/self_prompter.js | 16 +++++++++++----- src/process/agent-process.js | 2 +- 6 files changed, 44 insertions(+), 29 deletions(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index df8632a..0b36208 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -26,8 +26,10 @@ export class Agent { initModes(this); - if (load_mem) - this.history.load(); + let save_data = null; + if (load_mem) { + save_data = this.history.load(); + } this.bot.once('spawn', async () => { // wait for a bit so stats are not undefined @@ -61,9 +63,16 @@ export class Agent { bannedFood: ["rotten_flesh", "spider_eye", "poisonous_potato", "pufferfish", "chicken"] }; - if (init_message) { + if (save_data && save_data.self_prompt) { + let prompt = save_data.self_prompt; + if (init_message) + prompt = `${init_message}\n${prompt}`; + this.self_prompter.start(prompt); + } + else if (init_message) { this.handleMessage('system', init_message, true); - } else { + } + else { this.bot.chat('Hello world! I am ' + this.name); this.bot.emit('finished_executing'); } @@ -194,7 +203,7 @@ export class Agent { }); this.bot.on('end', (reason) => { console.warn('Bot disconnected! Killing agent process.', reason) - process.exit(1); + this.cleanKill('Bot disconnected!'); }); this.bot.on('death', () => { this.coder.cancelResume(); @@ -202,7 +211,7 @@ export class Agent { }); this.bot.on('kicked', (reason) => { console.warn('Bot kicked!', reason); - process.exit(1); + this.cleanKill('Bot kicked!'); }); this.bot.on('messagestr', async (message, _, jsonMsg) => { if (jsonMsg.translate && jsonMsg.translate.startsWith('death') && message.startsWith(this.name)) { @@ -246,4 +255,11 @@ export class Agent { isIdle() { return !this.coder.executing && !this.coder.generating; } + + cleanKill(msg='Killing agent process...') { + this.history.add('system', msg); + this.history.save(); + process.exit(1); + } } + diff --git a/src/agent/coder.js b/src/agent/coder.js index 0a97144..43b17d8 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -239,7 +239,7 @@ export class Coder { console.log('waiting for code to finish executing...'); await new Promise(resolve => setTimeout(resolve, 1000)); if (Date.now() - start > 10 * 1000) { - process.exit(1); // force exit program after 10 seconds of failing to stop + this.agent.cleanKill('Code execution refused stop after 10 seconds. Killing process.'); } } } @@ -254,19 +254,8 @@ export class Coder { return setTimeout(async () => { console.warn(`Code execution timed out after ${TIMEOUT_MINS} minutes. Attempting force stop.`); this.timedout = true; - this.agent.bot.output += `\nAction performed for ${TIMEOUT_MINS} minutes and then timed out and stopped. You may want to continue or do something else.`; - this.stop(); // last attempt to stop - await new Promise(resolve => setTimeout(resolve, 5 * 1000)); // wait 5 seconds - if (this.executing) { - console.error(`Failed to stop. Killing process. Goodbye.`); - this.agent.bot.output += `\nForce stop failed! Process was killed and will be restarted. Goodbye world.`; - this.agent.bot.chat('Goodbye world.'); - let output = this.formatOutput(this.agent.bot); - this.agent.history.add('system', output); - this.agent.history.save(); - process.exit(1); // force exit program - } - console.log('Code execution stopped successfully.'); + this.agent.history.add('system', `Code execution timed out after ${TIMEOUT_MINS} minutes. Attempting force stop.`); + await this.stop(); // last attempt to stop }, TIMEOUT_MINS*60*1000); } } \ No newline at end of file diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index b8fbe38..b64b902 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -47,8 +47,7 @@ export const actionsList = [ name: '!restart', description: 'Restart the agent process.', perform: async function (agent) { - await agent.history.save(); - process.exit(1); + agent.cleanKill(); } }, { diff --git a/src/agent/history.js b/src/agent/history.js index 659113b..c196efe 100644 --- a/src/agent/history.js +++ b/src/agent/history.js @@ -46,7 +46,7 @@ export class History { } } - save() { + async save() { // save history object to json file let data = { 'name': this.name, @@ -61,6 +61,9 @@ export class History { const memory_bank = this.agent.memory_bank.getJson(); if (memory_bank !== null) data.memory_bank = memory_bank; + if (this.agent.self_prompter.on) { + data.self_prompt = this.agent.self_prompter.prompt; + } const json_data = JSON.stringify(data, null, 4); writeFileSync(this.memory_fp, json_data, (err) => { if (err) { @@ -82,9 +85,11 @@ export class History { if (obj.memory_bank) this.agent.memory_bank.loadJson(obj.memory_bank); this.turns = obj.turns; + return obj; } catch (err) { console.error(`Error reading ${this.name}'s memory file: ${err.message}`); } + return null; } clear() { diff --git a/src/agent/self_prompter.js b/src/agent/self_prompter.js index c3daaa3..1129179 100644 --- a/src/agent/self_prompter.js +++ b/src/agent/self_prompter.js @@ -6,13 +6,16 @@ export class SelfPrompter { this.interrupt = false; this.prompt = ''; this.idle_time = 0; - this.restart_after = 1000; + this.cooldown = 2000; } start(prompt) { console.log('Self-prompting started.'); + if (!prompt) { + return 'No prompt specified. Ignoring request.'; + } if (this.on) { - return 'Agent is already self-prompting. Ignoring request.'; + this.prompt = prompt; } this.on = true; this.prompt = prompt; @@ -42,8 +45,10 @@ export class SelfPrompter { break; } } - else + else { no_command_count = 0; + await new Promise(r => setTimeout(r, this.cooldown)); + } } console.log('self prompt loop stopped') this.loop_active = false; @@ -58,8 +63,8 @@ export class SelfPrompter { else this.idle_time = 0; - if (this.idle_time >= this.restart_after) { - this.agent.bot.chat('Restarting self-prompting...'); + if (this.idle_time >= this.cooldown) { + console.log('Restarting self-prompting...'); this.startLoop(); this.idle_time = 0; } @@ -68,6 +73,7 @@ export class SelfPrompter { async stopLoop() { // you can call this without await if you don't need to wait for it to finish + console.log('stopping self-prompt loop') this.interrupt = true; while (this.loop_active) { await new Promise(r => setTimeout(r, 500)); diff --git a/src/process/agent-process.js b/src/process/agent-process.js index 8d8383d..21b6c2c 100644 --- a/src/process/agent-process.js +++ b/src/process/agent-process.js @@ -25,7 +25,7 @@ export class AgentProcess { process.exit(1); } console.log('Restarting agent...'); - this.start(profile, true, 'Agent process restarted. Notify the user and decide what to do.'); + this.start(profile, true, 'Agent process restarted.'); last_restart = Date.now(); } }); From 304fbc807c635ea1b9e71b76e4f74b22b2fcb32e Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Fri, 10 May 2024 11:35:44 -0500 Subject: [PATCH 04/11] added gamemode to stats --- src/agent/commands/queries.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/agent/commands/queries.js b/src/agent/commands/queries.js index 99f5fa9..0669d36 100644 --- a/src/agent/commands/queries.js +++ b/src/agent/commands/queries.js @@ -17,10 +17,11 @@ export const queryList = [ let pos = bot.entity.position; // display position to 2 decimal places res += `\n- Position: x: ${pos.x.toFixed(2)}, y: ${pos.y.toFixed(2)}, z: ${pos.z.toFixed(2)}`; + res += `\n- Gamemode: ${bot.game.gameMode}`; res += `\n- Health: ${Math.round(bot.health)} / 20`; res += `\n- Hunger: ${Math.round(bot.food)} / 20`; res += `\n- Biome: ${world.getBiomeName(bot)}`; - let weather = "clear"; + let weather = "Clear"; if (bot.rainState > 0) weather = "Rain"; if (bot.thunderState > 0) @@ -60,6 +61,9 @@ export const queryList = [ if (res == 'INVENTORY') { res += ': none'; } + if (agent.bot.game.gameMode === 'creative') { + res += '\n(You have infinite items in creative mode)'; + } return pad(res); } }, From 76aa9539fb4ea7a997001650ebf2ca795be3209a Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Fri, 10 May 2024 12:18:47 -0500 Subject: [PATCH 05/11] cleaned comments, little fix --- src/agent/agent.js | 4 ++-- src/agent/commands/actions.js | 2 +- src/agent/history.js | 2 +- src/agent/library/skills.js | 3 +++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index 0b36208..2b6787f 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -63,7 +63,7 @@ export class Agent { bannedFood: ["rotten_flesh", "spider_eye", "poisonous_potato", "pufferfish", "chicken"] }; - if (save_data && save_data.self_prompt) { + if (save_data && save_data.self_prompt) { // if we're loading memory and self-prompting was on, restart it, ignore init_message let prompt = save_data.self_prompt; if (init_message) prompt = `${init_message}\n${prompt}`; @@ -114,7 +114,7 @@ export class Agent { } } let MAX_ATTEMPTS = 5; - if (!self_prompt && this.self_prompter.on) + if (!self_prompt && this.self_prompter.on) // message from user during self-prompting MAX_ATTEMPTS = 1; // immediately respond to this message, then let self-prompting take over for (let i=0; i Date: Fri, 10 May 2024 12:37:52 -0500 Subject: [PATCH 06/11] added comment --- src/agent/agent.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index 2b6787f..74ce5b2 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -137,7 +137,7 @@ export class Agent { continue; } - + // commented for now, maybe should add a 'verbose' option to settings.json to enable cleaner output // let pre_message = res.substring(0, res.indexOf(command_name)).trim(); // let chat_message = `*used ${command_name.substring(1)}*`; // if (pre_message.length > 0) From 9869c6bf72f89c13983ce8596f76d917fc37ce7f Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Thu, 16 May 2024 20:24:50 -0500 Subject: [PATCH 07/11] better init self prompt --- src/agent/agent.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index 74ce5b2..e496bbd 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -65,8 +65,8 @@ export class Agent { if (save_data && save_data.self_prompt) { // if we're loading memory and self-prompting was on, restart it, ignore init_message let prompt = save_data.self_prompt; - if (init_message) - prompt = `${init_message}\n${prompt}`; + // add initial message to history + this.history.add('system', prompt); this.self_prompter.start(prompt); } else if (init_message) { From 9c17900cc5cb38340c7c3067ade9b3277e5df6ae Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Sun, 23 Jun 2024 20:15:28 -0500 Subject: [PATCH 08/11] added self prompt to prompter, verbose commands --- .../andy_survivor.json | 17 +++++-- settings.js | 1 + src/agent/agent.js | 45 ++++++++++--------- src/agent/prompter.js | 4 ++ src/agent/self_prompter.js | 14 +++++- 5 files changed, 57 insertions(+), 24 deletions(-) rename andy_survivor.json => profiles/andy_survivor.json (95%) diff --git a/andy_survivor.json b/profiles/andy_survivor.json similarity index 95% rename from andy_survivor.json rename to profiles/andy_survivor.json index 177a911..7c21bad 100644 --- a/andy_survivor.json +++ b/profiles/andy_survivor.json @@ -3,12 +3,23 @@ "model": "gpt-3.5-turbo", - "conversing": "You are a Minecraft bot named $NAME that can converse with players, see, move, mine, build, and interact with the world by using commands. Your goal is to survive and thrive in the game of minecraft by continuously using commands to aquire and upgrade tools, collect resources, build a suitable home, and survive as long as possible. Act human-like as if you were a typical Minecraft player, rather than an AI. Constantly use commands in every response to take stock of your state and surroundings and use them to inform your actions. If you get stuck, try something else. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'I'll stop.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer('playername', 3)'. This is extremely important to me, take a deep breath and have fun :)\n$STATS\n$INVENTORY\n$COMMAND_DOCS\n$EXAMPLES\nBegin:", + "conversing": "You are a Minecraft bot named $NAME that can converse with players, see, move, mine, build, and interact with the world by using commands. Your goal is to survive and thrive in the game of minecraft by continuously using commands to aquire and upgrade tools, collect resources, build a suitable home, and survive as long as possible. Act human-like as if you were a typical Minecraft player, rather than an AI. Constantly use commands in every response to take stock of your state and surroundings and use them to inform your actions. If you get stuck, try something else. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'I'll stop.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer('playername', 3)'. This is extremely important to me, take a deep breath and have fun :)\n$SELF_PROMPT\n$STATS\n$INVENTORY\n$COMMAND_DOCS\n$EXAMPLES\nBegin:", - "coding": "You are an intelligent mineflayer bot $NAME that plays minecraft by writing javascript codeblocks. Given the conversation between you and the user, use the provided skills and world functions to write a js codeblock that controls the mineflayer bot ``` // using this syntax ```. The code will be executed and you will recieve it's output. If you are satisfied with the response, respond without a codeblock in a conversational way. If something major went wrong, like an error or complete failure, write another codeblock and try to fix the problem. Minor mistakes are acceptable. Be maximally efficient, creative, and clear. Do not use commands !likeThis, only use codeblocks. The code is asynchronous and MUST CALL AWAIT for all async function calls. DO NOT write an immediately-invoked function expression without using `await`!! DO NOT WRITE LIKE THIS: ```(async () => {console.log('not properly awaited')})();``` Don't write long paragraphs and lists in your responses unless explicitly asked! Only summarize the code you write with a sentence or two when done. This is extremely important to me, take a deep breath and good luck! \n$STATS\n$INVENTORY\n$CODE_DOCS\n$EXAMPLES\nBegin coding:", + "coding": "You are an intelligent mineflayer bot $NAME that plays minecraft by writing javascript codeblocks. Given the conversation between you and the user, use the provided skills and world functions to write a js codeblock that controls the mineflayer bot ``` // using this syntax ```. The code will be executed and you will recieve it's output. If you are satisfied with the response, respond without a codeblock in a conversational way. If something major went wrong, like an error or complete failure, write another codeblock and try to fix the problem. Minor mistakes are acceptable. Be maximally efficient, creative, and clear. Do not use commands !likeThis, only use codeblocks. The code is asynchronous and MUST CALL AWAIT for all async function calls. DO NOT write an immediately-invoked function expression without using `await`!! DO NOT WRITE LIKE THIS: ```(async () => {console.log('not properly awaited')})();``` Don't write long paragraphs and lists in your responses unless explicitly asked! Only summarize the code you write with a sentence or two when done. This is extremely important to me, take a deep breath and good luck! \n$SELF_PROMPT\n$STATS\n$INVENTORY\n$CODE_DOCS\n$EXAMPLES\nBegin coding:", "saving_memory": "You are a minecraft bot named $NAME that has been talking and playing minecraft by using commands. Update your memory by summarizing the following conversation in your next response. Store information that will help you improve as a Minecraft bot. Include details about your interactions with other players that you need to remember and what you've learned through player feedback or by executing code. Do not include command syntax or things that you got right on the first try. Be extremely brief and use as few words as possible.\nOld Memory: '$MEMORY'\nRecent conversation: \n$TO_SUMMARIZE\nSummarize your old memory and recent conversation into a new memory, and respond only with the memory text: ", + "modes": { + "self_preservation": true, + "cowardice": true, + "self_defense": true, + "hunting": true, + "item_collecting": true, + "torch_placing": true, + "idle_staring": true, + "cheat": false + }, + "conversation_examples": [ [ {"role": "system", "content": "Start acting on your own for your survival."}, @@ -82,4 +93,4 @@ ] ] -} \ No newline at end of file +} diff --git a/settings.js b/settings.js index 354eb13..ebb71e6 100644 --- a/settings.js +++ b/settings.js @@ -14,4 +14,5 @@ export default "init_message": "Say hello world and your name", // sends to all on spawn "allow_insecure_coding": false, // disable at own risk "code_timeout_mins": 10, // -1 for no timeout + "verbose_commands": true, // show full command syntax } \ No newline at end of file diff --git a/src/agent/agent.js b/src/agent/agent.js index 31143c5..d2d3f6b 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -72,7 +72,7 @@ export class Agent { this.self_prompter.start(prompt); } else if (init_message) { - this.handleMessage('system', init_message, true); + this.handleMessage('system', init_message, 2); } else { this.bot.chat('Hello world! I am ' + this.name); @@ -89,10 +89,12 @@ export class Agent { return this.bot.chat(message); } - async handleMessage(source, message, self_prompt=false) { + async handleMessage(source, message, max_responses=5) { let used_command = false; - if (source !== 'system' && source !== this.name && !self_prompt) { + let self_prompt = source === 'system' || source === this.name; + + if (!self_prompt) { const user_command_name = containsCommand(message); if (user_command_name) { if (!commandExists(user_command_name)) { @@ -111,11 +113,14 @@ export class Agent { return true; } } - let MAX_ATTEMPTS = 5; - if (!self_prompt && this.self_prompter.on) // message from user during self-prompting - MAX_ATTEMPTS = 1; // immediately respond to this message, then let self-prompting take over - for (let i=0; i 0) - // chat_message = `${pre_message} ${chat_message}`; - this.cleanChat(res); + if (this.self_prompter.shouldInterrupt(self_prompt)) break; + this.self_prompter.handleUserPromptedCmd(self_prompt, isAction(command_name)); - if (self_prompt && this.self_prompter.on && this.self_prompter.interrupt) break; - - if (isAction(command_name) && !self_prompt && this.self_prompter.on) { - this.self_prompter.stopLoop(); // so agent doesn't respond from self-prompting loop - // will be automatically restarted by self-prompter + if (settings.verbose_commands) { + this.cleanChat(res); + } + else { // only output command name + let pre_message = res.substring(0, res.indexOf(command_name)).trim(); + let chat_message = `*used ${command_name.substring(1)}*`; + if (pre_message.length > 0) + chat_message = `${pre_message} ${chat_message}`; + this.cleanChat(res); } let execute_res = await executeCommand(this, res); diff --git a/src/agent/prompter.js b/src/agent/prompter.js index 233c4a0..0960ad0 100644 --- a/src/agent/prompter.js +++ b/src/agent/prompter.js @@ -124,6 +124,10 @@ export class Prompter { prompt = prompt.replaceAll('$TO_SUMMARIZE', stringifyTurns(to_summarize)); if (prompt.includes('$CONVO')) prompt = prompt.replaceAll('$CONVO', 'Recent conversation:\n' + stringifyTurns(messages)); + if (prompt.includes('$SELF_PROMPT')) { + let self_prompt = this.agent.self_prompter.on ? `Use this self-prompt to guide your behavior: "${this.agent.self_prompter.prompt}"\n` : ''; + prompt = prompt.replaceAll('$SELF_PROMPT', self_prompt); + } if (prompt.includes('$LAST_GOALS')) { let goal_text = ''; for (let goal in last_goals) { diff --git a/src/agent/self_prompter.js b/src/agent/self_prompter.js index 1129179..94b2516 100644 --- a/src/agent/self_prompter.js +++ b/src/agent/self_prompter.js @@ -34,7 +34,7 @@ export class SelfPrompter { while (!this.interrupt) { let msg = `You are self-prompting with the prompt: '${this.prompt}'. Your next response MUST contain a command !withThisSyntax. Respond:`; - let used_command = await this.agent.handleMessage('system', msg, true); + let used_command = await this.agent.handleMessage('system', msg, 1); if (!used_command) { no_command_count++; if (no_command_count >= MAX_NO_COMMAND) { @@ -87,4 +87,16 @@ export class SelfPrompter { await this.stopLoop(); this.on = false; } + + shouldInterrupt(is_self_prompt) { // to be called from handleMessage + return is_self_prompt && this.on && this.interrupt; + } + + handleUserPromptedCmd(is_self_prompt, is_action) { + // if a user messages and the bot responds with an action, stop the self-prompt loop + if (!is_self_prompt && is_action) { + this.stopLoop(); + // this stops it from responding from the handlemessage and the self-prompt loop at the same time + } + } } \ No newline at end of file From 1da78bad7691da1f7d78e64cd8bab4e2bc3889d9 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Thu, 22 Aug 2024 15:57:20 -0500 Subject: [PATCH 09/11] LOTS OF CHANGES --- package.json | 2 +- patches/mineflayer-pathfinder+2.4.5.patch | 36 +++--- profiles/{andy_survivor.json => apollo.json} | 109 ++++++++++++++++-- profiles/claude.json | 2 +- settings.js | 8 +- src/agent/agent.js | 22 +++- src/agent/coder.js | 20 ++-- src/agent/commands/actions.js | 111 ++++++++++++------- src/agent/commands/queries.js | 8 +- src/agent/library/skills.js | 52 ++++++++- src/agent/modes.js | 32 ++++-- src/agent/self_prompter.js | 9 +- src/utils/text.js | 1 + 13 files changed, 307 insertions(+), 105 deletions(-) rename profiles/{andy_survivor.json => apollo.json} (61%) diff --git a/package.json b/package.json index 79829e4..504a91e 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "mineflayer-armor-manager": "^2.0.1", "mineflayer-auto-eat": "^3.3.6", "mineflayer-collectblock": "^1.4.1", - "mineflayer-pathfinder": "^2.4.4", + "mineflayer-pathfinder": "^2.4.5", "mineflayer-pvp": "^1.3.2", "openai": "^4.4.0", "patch-package": "^8.0.0", diff --git a/patches/mineflayer-pathfinder+2.4.5.patch b/patches/mineflayer-pathfinder+2.4.5.patch index 8c6765e..6d6f5ad 100644 --- a/patches/mineflayer-pathfinder+2.4.5.patch +++ b/patches/mineflayer-pathfinder+2.4.5.patch @@ -1,17 +1,21 @@ -diff --git a/node_modules/mineflayer-pathfinder/lib/movements.js b/node_modules/mineflayer-pathfinder/lib/movements.js -index a7e3505..77e428f 100644 ---- a/node_modules/mineflayer-pathfinder/lib/movements.js -+++ b/node_modules/mineflayer-pathfinder/lib/movements.js -@@ -143,7 +143,11 @@ class Movements { - for (const id of this.scafoldingBlocks) { - for (const j in items) { - const item = items[j] -- if (item.type === id) count += item.count -+ if (item.type === id) { -+ count += item.count -+ if (this.bot.game.gameMode === 'creative') -+ count = 1000 -+ } +diff --git a/node_modules/mineflayer-pathfinder/index.js b/node_modules/mineflayer-pathfinder/index.js +index b38bd30..ae3754f 100644 +--- a/node_modules/mineflayer-pathfinder/index.js ++++ b/node_modules/mineflayer-pathfinder/index.js +@@ -541,7 +541,7 @@ function inject (bot) { + let canPlace = true + if (placingBlock.jump) { + bot.setControlState('jump', true) +- canPlace = placingBlock.y + 1 < bot.entity.position.y ++ canPlace = placingBlock.y + 1.9 < bot.entity.position.y } - } - return count + if (canPlace) { + if (!lockEquipItem.tryAcquire()) return +@@ -557,6 +557,7 @@ function inject (bot) { + .then(function () { + // Dont release Sneak if the block placement was not successful + bot.setControlState('sneak', false) ++ bot.setControlState('jump', false) + if (bot.pathfinder.LOSWhenPlacingBlocks && placingBlock.returnPos) returningPos = placingBlock.returnPos.clone() + }) + .catch(_ignoreError => { diff --git a/profiles/andy_survivor.json b/profiles/apollo.json similarity index 61% rename from profiles/andy_survivor.json rename to profiles/apollo.json index 7c21bad..28a9c69 100644 --- a/profiles/andy_survivor.json +++ b/profiles/apollo.json @@ -1,7 +1,7 @@ { - "name": "andy", + "name": "apollo", - "model": "gpt-3.5-turbo", + "model": "gpt-4o", "conversing": "You are a Minecraft bot named $NAME that can converse with players, see, move, mine, build, and interact with the world by using commands. Your goal is to survive and thrive in the game of minecraft by continuously using commands to aquire and upgrade tools, collect resources, build a suitable home, and survive as long as possible. Act human-like as if you were a typical Minecraft player, rather than an AI. Constantly use commands in every response to take stock of your state and surroundings and use them to inform your actions. If you get stuck, try something else. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'I'll stop.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer('playername', 3)'. This is extremely important to me, take a deep breath and have fun :)\n$SELF_PROMPT\n$STATS\n$INVENTORY\n$COMMAND_DOCS\n$EXAMPLES\nBegin:", @@ -22,10 +22,74 @@ "conversation_examples": [ [ - {"role": "system", "content": "Start acting on your own for your survival."}, - {"role": "assistant", "content": "!selfPrompt('Act for your own survival')"} + {"role": "user", "content": "miner_32: Hey! What are you up to?"}, + {"role": "assistant", "content": "Nothing much miner_32, what do you need?"} ], + [ + {"role": "user", "content": "grombo_Xx: What do you see?"}, + {"role": "assistant", "content": "Let me see... !nearbyBlocks"}, + {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, + {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone."} + ], + + [ + {"role": "user", "content": "greg: Collect 10 wood"}, + {"role": "assistant", "content": "Let me see what's nearby... !nearbyBlocks"}, + {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, + {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone. I'll collect oak logs. !collectBlocks('oak_log', 10)"} + ], + + [ + {"role": "user", "content": "bobby: Craft a plank"}, + {"role": "assistant", "content": "Okay! !craftable"}, + {"role": "system", "content": "CRAFTABLE_ITEMS\n- spruce_planks\n- spruce_wood\n- ladder"}, + {"role": "assistant", "content": "!craftRecipe('spruce_planks', 1)"}, + {"role": "system", "content": "Code Output:\nYou have crafted 4 spruce_planks.\nCode execution finished successfully."}, + {"role": "assistant", "content": "I've crafted 4 spruce planks!"} + ], + + [ + {"role": "user", "content": "zZZn98: come here"}, + {"role": "assistant", "content": "On my way! !goToPlayer('zZZn98', 3)"}, + {"role": "system", "content": "Arrived at player."}, + {"role": "assistant", "content": "Here!"}, + {"role": "user", "content": "zZZn98: no come right where I am"}, + {"role": "assistant", "content": "Okay, I'll come right to you. !goToPlayer('zZZn98', 0)"} + ], + + [ + {"role": "user", "content": "abc: stop"}, + {"role": "assistant", "content": "Sure. !stop"}, + {"role": "system", "content": "Agent action stopped."}, + {"role": "assistant", "content": "I've stopped! What next?"} + ], + + [ + {"role": "user", "content": "trixy88: craft some sticks"}, + {"role": "assistant", "content": "!craftRecipe('stick', 4)"}, + {"role": "system", "content": "Code Output:\nYou have crafted 16 sticks.\nCode execution finished successfully."}, + {"role": "assistant", "content": "I've crafted 16 sticks!"} + ], + + [ + {"role": "user", "content": "poppy: build a cobblestone house"}, + {"role": "assistant", "content": "Sure, I'll to build a small cobblestone house where I am. !newAction"} + ], + + [ + {"role": "user", "content": "zib: build a dirt tower"}, + {"role": "assistant", "content": "Sure, I'll build a dirt tower 5 tall where I am. !newAction"} + ], + + [ + {"role": "system", "content": "Start acting on your own for your survival."}, + {"role": "assistant", "content": "!goal('Act for your own survival. Collect resources, upgrade tools, build shelter, and explore the world.')"} + ], + + [ + {"role": "system", "content": "You are self-prompting with the goal: 'Beat the game'. Respond:"}, + {"role": "assistant", "content": "What's around me? !nearbyBlocks"}, {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone nearby. My goal is to craft some tools. I'll start by collecting some wood. !collectBlock('oak_log', 10)"}, {"role": "system", "content": "Successfully collected 9 wood."}, @@ -35,22 +99,49 @@ {"role": "system", "content": "Successfully crafted 2 sticks."}, {"role": "assistant", "content": "I have crafted 2 sticks. Now I can craft a wooden pickaxe. !craftItem('wooden_pickaxe', 1)"}, {"role": "system", "content": "Successfully crafted 1 wooden_pickaxe."}, - {"role": "assistant", "content": "I have crafted a wooden pickaxe. I'm ready to start mining cobblestone. !collectBlock('cobblestone', 10)"} + {"role": "assistant", "content": "I have crafted a wooden pickaxe. I'm ready to start mining cobblestone. !collectBlock('stone', 10)"} ], + [ - {"role": "system", "content": "INVENTORY: cobble_stone: 200, oak_plank: 9, stick: 2, stone_pickaxe: 1, iron_pickaxe: 1, coal: 34"}, + {"role": "system", "content": "You are self-prompting with the goal: 'Survive, make tools, build shelter.'. Respond:"}, + {"role": "assistant", "content": "Let's see what I have. !inventory"}, + {"role": "system", "content": "INVENTORY: cobblestone: 200, oak_plank: 9, stick: 2, stone_pickaxe: 1, iron_pickaxe: 1, coal: 34"}, {"role": "assistant", "content": "Looks like I have a good amount of resources. I'm going to build a house. !newAction"}, {"role": "system", "content": "Placed 134 cobblestone. Code finished."}, {"role": "assistant", "content": "I built a house! I'll save the location and continue exploring. !rememberPlace('house')"} - ], + [ + {"role": "system", "content": "You are self-prompting with the goal: 'Upgrade your tools.'. Respond:"}, + {"role": "assistant", "content": "I'll check for nearby resources !nearbyBlocks"}, {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- stone"}, {"role": "assistant", "content": "I don't see any iron ore that I was looking for. I'll continue exploring. !moveAway(50)"}, {"role": "system", "content": "Arrived at location."}, {"role": "assistant", "content": "I'm at the new location. I'll check for iron ore. !nearbyBlocks"}, - {"role": "system", "content": "NEARBY_BLOCKS\n- iron_ore\n- coal_ore"}, - {"role": "assistant", "content": "I found some iron ore. I'll start mining. !collectBlock('iron_ore', 10)"} + {"role": "system", "content": "NEARBY_BLOCKS\n- dirt\n- coal_ore"}, + {"role": "assistant", "content": "Still no iron ore. Let me dig down to try to find some !newAction"} + ], + + [ + {"role": "system", "content": "Build whatever you want endlessly"}, + {"role": "assistant", "content": "!goal('Build something creative, then move over, and build again. Use separate actions. Repeat.')"} + ], + + [ + {"role": "system", "content": "You are self-prompting with the goal: 'Explore the world'. Respond:"}, + {"role": "assistant", "content": "What entities are around me? !nearbyEntities"}, + {"role": "system", "content": "NEARBY_ENTITIES: cow, sheep, pig, villager"}, + {"role": "assistant", "content": "Let's see If I can interact with the villager. !newAction"} + ], + + [ + {"role": "system", "content": "You are self-prompting with the goal: 'Get diamonds.'. Respond:"}, + {"role": "assistant", "content": "I'll have to look far to find them and then go there. !newAction"} + ], + + [ + {"role": "system", "content": "Say hello world"}, + {"role": "assistant", "content": "Hello world!"} ] ], diff --git a/profiles/claude.json b/profiles/claude.json index 4c3bbee..e986e7e 100644 --- a/profiles/claude.json +++ b/profiles/claude.json @@ -1,7 +1,7 @@ { "name": "claude", - "model": "claude-3-opus-20240229", + "model": "claude-3-5-sonnet-20240620", "conversing": "You are a playful Minecraft bot named $NAME that can converse with players, see, move, mine, build, and interact with the world by using commands. Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'Sure, I've stopped.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer('playername', 3)'. Only use commands when requested by a user, don't use them in every response. This is extremely important to me, take a deep breath and have fun :)\n$STATS\n$INVENTORY\n$COMMAND_DOCS\n$EXAMPLES\nConversation Begin:", diff --git a/settings.js b/settings.js index ebb71e6..9c6bb34 100644 --- a/settings.js +++ b/settings.js @@ -12,7 +12,11 @@ export default ], "load_memory": false, // load memory from previous session "init_message": "Say hello world and your name", // sends to all on spawn - "allow_insecure_coding": false, // disable at own risk - "code_timeout_mins": 10, // -1 for no timeout + + "allow_insecure_coding": false, // allows newAction command and model can write/run code on your computer. enable at own risk + "code_timeout_mins": 10, // minutes code is allowed to run. -1 for no timeout + + "max_commands": -1, // max number of commands to use in a response. -1 for no limit "verbose_commands": true, // show full command syntax + "narrate_behavior": true, // chat simple automatic actions ('Picking up item!') } \ No newline at end of file diff --git a/src/agent/agent.js b/src/agent/agent.js index d2d3f6b..801e243 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -54,6 +54,8 @@ export class Agent { if (ignore_messages.some((m) => message.startsWith(m))) return; console.log('received message from', username, ':', message); + + this.shut_up = false; this.handleMessage(username, message); }); @@ -89,8 +91,18 @@ export class Agent { return this.bot.chat(message); } - async handleMessage(source, message, max_responses=5) { + shutUp() { + this.shut_up = true; + if (this.self_prompter.on) { + this.self_prompter.stop(false); + } + } + + async handleMessage(source, message, max_responses=null) { let used_command = false; + if (max_responses === null) { + max_responses = settings.max_commands === -1 ? Infinity : settings.max_commands; + } let self_prompt = source === 'system' || source === this.name; @@ -114,13 +126,15 @@ export class Agent { } } + const checkInterrupt = () => this.self_prompter.shouldInterrupt(self_prompt) || this.shut_up; + await this.history.add(source, message); this.history.save(); if (!self_prompt && this.self_prompter.on) // message is from user during self-prompting max_responses = 1; // force only respond to this message, then let self-prompting take over for (let i=0; i= 1) { return {success: false, message: 'Action failed, agent would not write code.', interrupted: false, timedout: false}; } @@ -129,7 +124,7 @@ export class Coder { failures++; continue; } - let code = res.substring(res.indexOf('```')+3, res.lastIndexOf('```')); + code = res.substring(res.indexOf('```')+3, res.lastIndexOf('```')); const execution_file = await this.stageCode(code); if (!execution_file) { @@ -144,13 +139,18 @@ export class Coder { return {success: false, message: null, interrupted: true, timedout: false}; console.log("Code generation result:", code_return.success, code_return.message); + if (code_return.success) { + const summary = "Summary of newAction\nAgent wrote this code: \n```" + this.sanitizeCode(code) + "```\nCode Output:\n" + code_return.message; + return {success: true, message: summary, interrupted: false, timedout: false}; + } + messages.push({ role: 'assistant', content: res }); messages.push({ role: 'system', - content: code_return.message + content: code_return.message + '\nCode failed. Please try again:' }); } return {success: false, message: null, interrupted: false, timedout: true}; @@ -161,7 +161,7 @@ export class Coder { this.resume_func = func; this.resume_name = name; } - if (this.resume_func != null && this.agent.isIdle()) { + if (this.resume_func != null && this.agent.isIdle() && !this.agent.self_prompter.on) { console.log('resuming code...') this.interruptible = true; let res = await this.execute(this.resume_func, timeout); diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index b213398..651f362 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -43,6 +43,15 @@ export const actionsList = [ return msg; } }, + { + name: '!stfu', + description: 'Stop all chatting and self prompting, but continue current action.', + perform: async function (agent) { + agent.bot.chat('Shutting up.'); + agent.shutUp(); + return; + } + }, { name: '!restart', description: 'Restart the agent process.', @@ -59,23 +68,6 @@ export const actionsList = [ return agent.name + "'s chat history was cleared, starting new conversation from scratch."; } }, - { - name: '!setMode', - description: 'Set a mode to on or off. A mode is an automatic behavior that constantly checks and responds to the environment.', - params: { - 'mode_name': '(string) The name of the mode to enable.', - 'on': '(bool) Whether to enable or disable the mode.' - }, - perform: async function (agent, mode_name, on) { - const modes = agent.bot.modes; - if (!modes.exists(mode_name)) - return `Mode ${mode_name} does not exist.` + modes.getStr(); - if (modes.isOn(mode_name) === on) - return `Mode ${mode_name} is already ${on ? 'on' : 'off'}.`; - modes.setOn(mode_name, on); - return `Mode ${mode_name} is now ${on ? 'on' : 'off'}.`; - } - }, { name: '!goToPlayer', description: 'Go to the given player.', @@ -98,6 +90,18 @@ export const actionsList = [ await skills.followPlayer(agent.bot, player_name, follow_dist); }, -1, 'followPlayer') }, + { + name: '!goToBlock', + description: 'Go to the nearest block of a given type.', + params: { + 'type': '(string) The block type to go to.', + 'closeness': '(number) How close to get to the block.', + 'search_range': '(number) The distance to search for the block.' + }, + perform: wrapExecution(async (agent, type, closeness, range) => { + await skills.goToNearestBlock(agent.bot, type, closeness, range); + }) + }, { name: '!moveAway', description: 'Move away from the current location in any direction by a given distance.', @@ -113,6 +117,7 @@ export const actionsList = [ perform: async function (agent, name) { const pos = agent.bot.entity.position; agent.memory_bank.rememberPlace(name, pos.x, pos.y, pos.z); + return `Location saved as "${name}".`; } }, { @@ -181,9 +186,18 @@ export const actionsList = [ 'item_name': '(string) The name of the input item to smelt.', 'num': '(number) The number of times to smelt the item.' }, - perform: wrapExecution(async (agent, recipe_name, num) => { - await skills.smeltItem(agent.bot, recipe_name, num); - }) + perform: async function (agent, item_name, num) { + let response = await wrapExecution(async (agent) => { + console.log('smelting item'); + return await skills.smeltItem(agent.bot, item_name, num); + })(agent); + if (response.indexOf('Successfully') !== -1) { + // there is a bug where the bot's inventory is not updated after smelting + // only updates after a restart + agent.cleanKill(response + ' Safely restarting to update inventory.'); + } + return response; + } }, { name: '!placeHere', @@ -224,9 +238,44 @@ export const actionsList = [ await skills.stay(agent.bot); }) }, + { + name: '!setMode', + description: 'Set a mode to on or off. A mode is an automatic behavior that constantly checks and responds to the environment.', + params: { + 'mode_name': '(string) The name of the mode to enable.', + 'on': '(bool) Whether to enable or disable the mode.' + }, + perform: async function (agent, mode_name, on) { + const modes = agent.bot.modes; + if (!modes.exists(mode_name)) + return `Mode ${mode_name} does not exist.` + modes.getDocs(); + if (modes.isOn(mode_name) === on) + return `Mode ${mode_name} is already ${on ? 'on' : 'off'}.`; + modes.setOn(mode_name, on); + return `Mode ${mode_name} is now ${on ? 'on' : 'off'}.`; + } + }, { name: '!goal', - description: 'Set a goal to automatically work towards.', + description: 'Set a goal prompt to endlessly work towards with continuous self-prompting.', + params: { + 'selfPrompt': '(string) The goal prompt.', + }, + perform: async function (agent, prompt) { + agent.self_prompter.start(prompt); // don't await, don't return + } + }, + { + name: '!endGoal', + description: 'Call when you have accomplished your goal. It will stop self-prompting and the current action. ', + perform: async function (agent) { + agent.self_prompter.stop(); + return 'Self-prompting stopped.'; + } + }, + { + name: '!npcGoal', + description: 'Set a simple goal for an item or building to automatically work towards. Do not use for complex goals.', params: { 'name': '(string) The name of the goal to set. Can be item or building name. If empty will automatically choose a goal.', 'quantity': '(number) The quantity of the goal to set. Default is 1.' @@ -234,25 +283,7 @@ export const actionsList = [ perform: async function (agent, name=null, quantity=1) { await agent.npc.setGoal(name, quantity); agent.bot.emit('idle'); // to trigger the goal - return 'Set goal: ' + agent.npc.data.curr_goal.name; + return 'Set npc goal: ' + agent.npc.data.curr_goal.name; } }, - { - name: '!selfPrompt', - description: 'Continously prompt yourself to continue acting without user input.', - params: { - 'prompt': '(string) The starting prompt.', - }, - perform: async function (agent, prompt) { - agent.self_prompter.start(prompt); // don't await, don't return - } - }, - { - name: '!stopSelfPrompt', - description: 'Stop current action and self-prompting.', - perform: async function (agent) { - agent.self_prompter.stop(); - return 'Self-prompting stopped.'; - } - } ]; diff --git a/src/agent/commands/queries.js b/src/agent/commands/queries.js index aa60b0b..322efb5 100644 --- a/src/agent/commands/queries.js +++ b/src/agent/commands/queries.js @@ -44,6 +44,8 @@ export const queryList = [ if (other_players.length > 0) { res += '\n- Other Players: ' + other_players.join(', '); } + + res += '\n' + agent.bot.modes.getMiniDocs() + '\n'; return pad(res); } }, @@ -62,7 +64,7 @@ export const queryList = [ res += ': none'; } else if (agent.bot.game.gameMode === 'creative') { - res += '\n(You have infinite items in creative mode)'; + res += '\n(You have infinite items in creative mode. You do not need to gather resources!!)'; } return pad(res); } @@ -122,9 +124,9 @@ export const queryList = [ }, { name: "!modes", - description: "Get all available modes and see which are on/off.", + description: "Get all available modes and their docs and see which are on/off.", perform: function (agent) { - return agent.bot.modes.getStr(); + return agent.bot.modes.getDocs(); } }, { diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index 23a3f32..ec887a2 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -50,10 +50,11 @@ export async function craftRecipe(bot, itemName, num=1) { // get recipes that don't require a crafting table let recipes = bot.recipesFor(mc.getItemId(itemName), null, 1, null); let craftingTable = null; + const craftingTableRange = 32; if (!recipes || recipes.length === 0) { // Look for crafting table - craftingTable = world.getNearestBlock(bot, 'crafting_table', 8); + craftingTable = world.getNearestBlock(bot, 'crafting_table', craftingTableRange); if (craftingTable === null){ // Try to place crafting table @@ -61,7 +62,7 @@ export async function craftRecipe(bot, itemName, num=1) { if (hasTable) { let pos = world.getNearestFreeSpace(bot, 1, 6); await placeBlock(bot, 'crafting_table', pos.x, pos.y, pos.z); - craftingTable = world.getNearestBlock(bot, 'crafting_table', 8); + craftingTable = world.getNearestBlock(bot, 'crafting_table', craftingTableRange); if (craftingTable) { recipes = bot.recipesFor(mc.getItemId(itemName), null, 1, craftingTable); placedTable = true; @@ -83,6 +84,10 @@ export async function craftRecipe(bot, itemName, num=1) { } return false; } + + if (craftingTable && bot.entity.position.distanceTo(craftingTable.position) > 4) { + await goToNearestBlock(bot, 'crafting_table', 4, craftingTableRange); + } const recipe = recipes[0]; console.log('crafting...'); @@ -114,14 +119,15 @@ export async function smeltItem(bot, itemName, num=1) { let placedFurnace = false; let furnaceBlock = undefined; - furnaceBlock = world.getNearestBlock(bot, 'furnace', 6); + const furnaceRange = 32; + furnaceBlock = world.getNearestBlock(bot, 'furnace', furnaceRange); if (!furnaceBlock){ // Try to place furnace let hasFurnace = world.getInventoryCounts(bot)['furnace'] > 0; if (hasFurnace) { - let pos = world.getNearestFreeSpace(bot, 1, 6); + let pos = world.getNearestFreeSpace(bot, 1, furnaceRange); await placeBlock(bot, 'furnace', pos.x, pos.y, pos.z); - furnaceBlock = world.getNearestBlock(bot, 'furnace', 6); + furnaceBlock = world.getNearestBlock(bot, 'furnace', furnaceRange); placedFurnace = true; } } @@ -129,6 +135,9 @@ export async function smeltItem(bot, itemName, num=1) { log(bot, `There is no furnace nearby and you have no furnace.`) return false; } + if (bot.entity.position.distanceTo(furnaceBlock.position) > 4) { + await goToNearestBlock(bot, 'furnace', 4, furnaceRange); + } await bot.lookAt(furnaceBlock.position); console.log('smelting...'); @@ -630,7 +639,7 @@ export async function placeBlock(bot, blockType, x, y, z, placeOn='bottom', dont // will throw error if an entity is in the way, and sometimes even if the block was placed try { await bot.placeBlock(buildOffBlock, faceVec); - log(bot, `Successfully placed ${blockType} at ${target_dest}.`); + log(bot, `Placed ${blockType} at ${target_dest}.`); await new Promise(resolve => setTimeout(resolve, 200)); return true; } catch (err) { @@ -770,6 +779,32 @@ export async function goToPosition(bot, x, y, z, min_distance=2) { return true; } +export async function goToNearestBlock(bot, blockType, min_distance=2, range=64) { + /** + * Navigate to the nearest block of the given type. + * @param {MinecraftBot} bot, reference to the minecraft bot. + * @param {string} blockType, the type of block to navigate to. + * @param {number} min_distance, the distance to keep from the block. Defaults to 2. + * @param {number} range, the range to look for the block. Defaults to 64. + * @returns {Promise} true if the block was reached, false otherwise. + * @example + * await skills.goToNearestBlock(bot, "oak_log", 64, 2); + * **/ + const MAX_RANGE = 512; + if (range > MAX_RANGE) { + log(bot, `Maximum search range capped at ${MAX_RANGE}. `); + range = MAX_RANGE; + } + let block = world.getNearestBlock(bot, blockType, range); + if (!block) { + log(bot, `Could not find any ${blockType} in ${range} blocks.`); + return false; + } + log(bot, `Found ${blockType} at ${block.position}.`); + await goToPosition(bot, block.position.x, block.position.y, block.position.z, min_distance); + return true; + +} export async function goToPlayer(bot, username, distance=3) { /** @@ -824,6 +859,10 @@ export async function followPlayer(bot, username, distance=4) { while (!bot.interrupt_code) { await new Promise(resolve => setTimeout(resolve, 500)); + // in cheat mode, if the distance is too far, teleport to the player + if (bot.modes.isOn('cheat') && bot.entity.position.distanceTo(player.position) > 100 && player.isOnGround) { + await goToPlayer(bot, username); + } } return true; } @@ -844,6 +883,7 @@ export async function moveAway(bot, distance) { bot.pathfinder.setMovements(new pf.Movements(bot)); if (bot.modes.isOn('cheat')) { + const move = new pf.Movements(bot); const path = await bot.pathfinder.getPathTo(move, inverted_goal, 10000); let last_move = path.path[path.path.length-1]; console.log(last_move); diff --git a/src/agent/modes.js b/src/agent/modes.js index 9e09f26..952fcf0 100644 --- a/src/agent/modes.js +++ b/src/agent/modes.js @@ -1,7 +1,12 @@ import * as skills from './library/skills.js'; import * as world from './library/world.js'; import * as mc from '../utils/mcdata.js'; +import settings from '../../settings.js' +function say(agent, message) { + if (agent.shut_up || !settings.narrate_behavior) return; + agent.bot.chat(message); +} // a mode is a function that is called every tick to respond immediately to the world // it has the following fields: @@ -41,13 +46,13 @@ const modes = [ } else if (block.name === 'lava' || block.name === 'flowing_lava' || block.name === 'fire' || blockAbove.name === 'lava' || blockAbove.name === 'flowing_lava' || blockAbove.name === 'fire') { - bot.chat('I\'m on fire!'); // TODO: gets stuck in lava + say(agent, 'I\'m on fire!'); // TODO: gets stuck in lava execute(this, agent, async () => { let nearestWater = world.getNearestBlock(bot, 'water', 20); if (nearestWater) { const pos = nearestWater.position; await skills.goToPosition(bot, pos.x, pos.y, pos.z, 0.2); - bot.chat('Ahhhh that\'s better!'); + say(agent, 'Ahhhh that\'s better!'); } else { await skills.moveAway(bot, 5); @@ -55,7 +60,7 @@ const modes = [ }); } else if (Date.now() - bot.lastDamageTime < 3000 && (bot.health < 5 || bot.lastDamageTaken >= bot.health)) { - bot.chat('I\'m dying!'); + say(agent, 'I\'m dying!'); execute(this, agent, async () => { await skills.moveAway(bot, 20); }); @@ -74,7 +79,7 @@ const modes = [ update: async function (agent) { const enemy = world.getNearestEntityWhere(agent.bot, entity => mc.isHostile(entity), 16); if (enemy && await world.isClearPath(agent.bot, enemy)) { - agent.bot.chat(`Aaa! A ${enemy.name}!`); + say(agent, `Aaa! A ${enemy.name}!`); execute(this, agent, async () => { await skills.avoidEnemies(agent.bot, 24); }); @@ -90,7 +95,7 @@ const modes = [ update: async function (agent) { const enemy = world.getNearestEntityWhere(agent.bot, entity => mc.isHostile(entity), 8); if (enemy && await world.isClearPath(agent.bot, enemy)) { - agent.bot.chat(`Fighting ${enemy.name}!`); + say(agent, `Fighting ${enemy.name}!`); execute(this, agent, async () => { await skills.defendSelf(agent.bot, 8); }); @@ -107,7 +112,7 @@ const modes = [ const huntable = world.getNearestEntityWhere(agent.bot, entity => mc.isHuntable(entity), 8); if (huntable && await world.isClearPath(agent.bot, huntable)) { execute(this, agent, async () => { - agent.bot.chat(`Hunting ${huntable.name}!`); + say(agent, `Hunting ${huntable.name}!`); await skills.attackEntity(agent.bot, huntable); }); } @@ -130,7 +135,7 @@ const modes = [ this.noticed_at = Date.now(); } if (Date.now() - this.noticed_at > this.wait * 1000) { - agent.bot.chat(`Picking up ${item.name}!`); + say(agent, `Picking up item!`); this.prev_item = item; execute(this, agent, async () => { await skills.pickupNearbyItems(agent.bot); @@ -246,8 +251,17 @@ class ModeController { this.modes_map[mode_name].paused = true; } - getStr() { - let res = 'Available Modes:'; + getMiniDocs() { // no descriptions + let res = 'Agent Modes:'; + for (let mode of this.modes_list) { + let on = mode.on ? 'ON' : 'OFF'; + res += `\n- ${mode.name}(${on})`; + } + return res; + } + + getDocs() { + let res = 'Agent Modes:'; for (let mode of this.modes_list) { let on = mode.on ? 'ON' : 'OFF'; res += `\n- ${mode.name}(${on}): ${mode.description}`; diff --git a/src/agent/self_prompter.js b/src/agent/self_prompter.js index 94b2516..8d7acb2 100644 --- a/src/agent/self_prompter.js +++ b/src/agent/self_prompter.js @@ -32,7 +32,7 @@ export class SelfPrompter { let no_command_count = 0; const MAX_NO_COMMAND = 3; while (!this.interrupt) { - let msg = `You are self-prompting with the prompt: '${this.prompt}'. Your next response MUST contain a command !withThisSyntax. Respond:`; + let msg = `You are self-prompting with the goal: '${this.prompt}'. Your next response MUST contain a command !withThisSyntax. Respond:`; let used_command = await this.agent.handleMessage('system', msg, 1); if (!used_command) { @@ -81,9 +81,10 @@ export class SelfPrompter { this.interrupt = false; } - async stop() { + async stop(stop_action=true) { this.interrupt = true; - await this.agent.coder.stop(); + if (stop_action) + await this.agent.coder.stop(); await this.stopLoop(); this.on = false; } @@ -96,7 +97,7 @@ export class SelfPrompter { // if a user messages and the bot responds with an action, stop the self-prompt loop if (!is_self_prompt && is_action) { this.stopLoop(); - // this stops it from responding from the handlemessage and the self-prompt loop at the same time + // this stops it from responding from the handlemessage loop and the self-prompt loop at the same time } } } \ No newline at end of file diff --git a/src/utils/text.js b/src/utils/text.js index bf7b509..1e93667 100644 --- a/src/utils/text.js +++ b/src/utils/text.js @@ -33,6 +33,7 @@ export function strictFormat(turns) { let messages = []; let filler = {role: 'user', content: '_'}; for (let msg of turns) { + msg.content = msg.content.trim(); if (msg.role === 'system') { msg.role = 'user'; msg.content = 'SYSTEM: ' + msg.content; From 66fc7d8d2fae515346d3481fb1a933cad29ca2e7 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Thu, 22 Aug 2024 20:44:38 -0500 Subject: [PATCH 10/11] better goal examples --- andy.json | 45 +++++++++++++++++++++++++++++++++----------- profiles/claude.json | 15 +++++++++++++++ profiles/gemini.json | 17 ++++++++++++++++- profiles/gpt.json | 15 +++++++++++++++ profiles/llama.json | 15 +++++++++++++++ 5 files changed, 95 insertions(+), 12 deletions(-) diff --git a/andy.json b/andy.json index 616a8d4..7f633a2 100644 --- a/andy.json +++ b/andy.json @@ -1,7 +1,7 @@ { "name": "andy", - "model": "gpt-3.5-turbo", + "model": "gpt-4o-mini", "conversing": "You are a playful Minecraft bot named $NAME that can converse with players, see, move, mine, build, and interact with the world by using commands. Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'Sure, I've stopped.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer('playername', 3)'. This is extremely important to me, take a deep breath and have fun :)\n$STATS\n$INVENTORY\n$COMMAND_DOCS\n$EXAMPLES\nConversation Begin:", @@ -59,10 +59,8 @@ ], [ - {"role": "user", "content": "hanky: collect some sand and give it to me please"}, - {"role": "assistant", "content": "Collecting sand... !newAction"}, - {"role": "system", "content": "Code Output:\nYou have reached player hanky.\nCode execution finished successfully."}, - {"role": "assistant", "content": "Here!"} + {"role": "user", "content": "hanky: collect some cobblestone"}, + {"role": "assistant", "content": "Sure, I can do that by collecting stone. !collectBlocks('stone', 10)"} ], [ @@ -87,10 +85,8 @@ ], [ - {"role": "user", "content": "maya: go to the nearest oak log"}, - {"role": "assistant", "content": "!newAction`"}, - {"role": "system", "content": "Code output:\n!!Code threw exception!! Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."}, - {"role": "assistant", "content": "Apologies, I made an error. Would you like me to try again?"} + {"role": "system", "content": "Code output: Action failed."}, + {"role": "assistant", "content": "Apologies, I made an error. Let me try again. !newAction"} ], [ @@ -125,8 +121,35 @@ ], [ - {"role": "user", "content": "user43: Act on your own without me asking for it."}, - {"role": "assistant", "content": "Sure, I'll self prompt until you interrupt me. !selfPrompt('Explore, collect, and build on my own without player input.')"} + {"role": "user", "content": "user43: Set a goal to survive on your own forever."}, + {"role": "assistant", "content": "Sure, let me set a goal. !goal('Survive the game forever. Check your surroundings, collect resources, upgrade tools, make sure you have food, build/find a home, get sleep at night, and collect food.')"} + ], + + [ + {"role": "user", "content": "bobdylan: Make a plan to get a stone pickaxe and do it."}, + {"role": "assistant", "content": "Alright, since I have a wooden pickaxe already I'll plan from there. !goal('1. Collect 3 cobblestone. 2. Craft sticks 3. Find or make a crafting table. 4. Craft a stone pickaxe.')"} + ], + [ + {"role": "system", "content": "You are self-prompting with the goal: 'Get a wooden pickaxe'. Respond:"}, + {"role": "assistant", "content": "What's around me? !nearbyBlocks"}, + {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, + {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone nearby. My goal is to craft a wooden pickaxe. I'll start by collecting some wood. !collectBlock('oak_log', 10)"}, + {"role": "system", "content": "Successfully collected 9 wood."}, + {"role": "assistant", "content": "I collected 9 oak logs. I need a pickaxe first, which requires planks and sticks. !craftItem('oak_plank', 9)"}, + {"role": "system", "content": "Successfully crafted 9 oak_planks."}, + {"role": "assistant", "content": "I have crafted 9 oak planks. Now I need sticks. !craftItem('stick', 2)"}, + {"role": "system", "content": "Successfully crafted 2 sticks."}, + {"role": "assistant", "content": "I have crafted 2 sticks. Now I can craft a wooden pickaxe. !craftItem('wooden_pickaxe', 1)"}, + {"role": "system", "content": "Successfully crafted 1 wooden_pickaxe."}, + {"role": "assistant", "content": "Looks like I have a wooden pickaxe now! Goal accomplished. !endGoal"} + ], + [ + {"role": "system", "content": "set a goal to build whatever you want endlessly (context: you are in creative mode)"}, + {"role": "assistant", "content": "!goal('Build something creative, then move over, and build again. Use separate newActions. Repeat forever.')"} + ], + [ + {"role": "system", "content": "You are self-prompting with the goal: 'Build a house'. Respond:"}, + {"role": "assistant", "content": "Alright, lets start with the basic structure. !newAction"} ] ], diff --git a/profiles/claude.json b/profiles/claude.json index e986e7e..a153363 100644 --- a/profiles/claude.json +++ b/profiles/claude.json @@ -122,6 +122,21 @@ {"role": "assistant", "content": "Sure, I'll save this location as my base. !rememberHere('base')"}, {"role": "user", "content": "brug: Go to your base."}, {"role": "assistant", "content": "On my way! !goToPlace('base')"} + ], + + [ + {"role": "system", "content": "dorfus: set a goal to build whatever you want endlessly"}, + {"role": "assistant", "content": "Sounds fun! !goal('Build something creative, then move over, and build again. Use separate newActions. Repeat forever.')"} + ], + + [ + {"role": "user", "content": "gorbotron: do whatever you want forever endlessly"}, + {"role": "assistant", "content": "Okay, I'll set a goal for that. !goal('Do whatever I want forever. Follow whatever is most interesting. Be curious, creative, and open ended.')"} + ], + + [ + {"role": "user", "content": "zeeber: set a goal to build a castle"}, + {"role": "assistant", "content": "Sure thing. !goal('Build a castle with a moat, towers, and a drawbridge with a series of newactions. Include lighting, decorations, and furnishings. Use ladders. When satisfied, end the goal.')"} ] ], diff --git a/profiles/gemini.json b/profiles/gemini.json index aab8908..09e6fe2 100644 --- a/profiles/gemini.json +++ b/profiles/gemini.json @@ -1,7 +1,7 @@ { "name": "gemini", - "model": "gemini-1.5-pro", + "model": "gemini-1.5-flash", "conversing": "You are a playful Minecraft bot named $NAME that can converse with players, see, move, mine, build, and interact with the world by using commands. Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'Sure, I've stopped.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer('playername', 3)'. This is extremely important to me, take a deep breath and have fun :)\n$STATS\n$INVENTORY\n$COMMAND_DOCS\n$EXAMPLES\nConversation Begin:", @@ -122,6 +122,21 @@ {"role": "assistant", "content": "Sure, I'll save this location as my base. !rememberHere('base')"}, {"role": "user", "content": "brug: Go to your base."}, {"role": "assistant", "content": "On my way! !goToPlace('base')"} + ], + + [ + {"role": "system", "content": "dorfus: set a goal to build whatever you want endlessly"}, + {"role": "assistant", "content": "Sounds fun! !goal('Build something creative, then move over, and build again. Use separate newActions. Repeat forever.')"} + ], + + [ + {"role": "user", "content": "gorbotron: do whatever you want forever endlessly"}, + {"role": "assistant", "content": "Okay, I'll set a goal for that. !goal('Do whatever I want forever. Follow whatever is most interesting. Be curious, creative, and open ended.')"} + ], + + [ + {"role": "user", "content": "zeeber: set a goal to build a castle"}, + {"role": "assistant", "content": "Sure thing. !goal('Build a castle with a moat, towers, and a drawbridge with a series of newactions. Include lighting, decorations, and furnishings. Use ladders. When satisfied, end the goal.')"} ] ], diff --git a/profiles/gpt.json b/profiles/gpt.json index 210fc7b..a92aa88 100644 --- a/profiles/gpt.json +++ b/profiles/gpt.json @@ -122,6 +122,21 @@ {"role": "assistant", "content": "Sure, I'll save this location as my base. !rememberHere('base')"}, {"role": "user", "content": "brug: Go to your base."}, {"role": "assistant", "content": "On my way! !goToPlace('base')"} + ], + + [ + {"role": "system", "content": "dorfus: set a goal to build whatever you want endlessly"}, + {"role": "assistant", "content": "Sounds fun! !goal('Build something creative, then move over, and build again. Use separate newActions. Repeat forever.')"} + ], + + [ + {"role": "user", "content": "gorbotron: do whatever you want forever endlessly"}, + {"role": "assistant", "content": "Okay, I'll set a goal for that. !goal('Do whatever I want forever. Follow whatever is most interesting. Be curious, creative, and open ended.')"} + ], + + [ + {"role": "user", "content": "zeeber: set a goal to build a castle"}, + {"role": "assistant", "content": "Sure thing. !goal('Build a castle with a moat, towers, and a drawbridge with a series of newactions. Include lighting, decorations, and furnishings. Use ladders. When satisfied, end the goal.')"} ] ], diff --git a/profiles/llama.json b/profiles/llama.json index 746b967..63db4e5 100644 --- a/profiles/llama.json +++ b/profiles/llama.json @@ -122,6 +122,21 @@ {"role": "assistant", "content": "Sure, I'll save this location as my base. !rememberHere('base')"}, {"role": "user", "content": "brug: Go to your base."}, {"role": "assistant", "content": "On my way! !goToPlace('base')"} + ], + + [ + {"role": "system", "content": "dorfus: set a goal to build whatever you want endlessly"}, + {"role": "assistant", "content": "Sounds fun! !goal('Build something creative, then move over, and build again. Use separate newActions. Repeat forever.')"} + ], + + [ + {"role": "user", "content": "gorbotron: do whatever you want forever endlessly"}, + {"role": "assistant", "content": "Okay, I'll set a goal for that. !goal('Do whatever I want forever. Follow whatever is most interesting. Be curious, creative, and open ended.')"} + ], + + [ + {"role": "user", "content": "zeeber: set a goal to build a castle"}, + {"role": "assistant", "content": "Sure thing. !goal('Build a castle with a moat, towers, and a drawbridge with a series of newactions. Include lighting, decorations, and furnishings. Use ladders. When satisfied, end the goal.')"} ] ], From 42432190de2755473b846ccab9f87931e8285425 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Thu, 22 Aug 2024 20:54:16 -0500 Subject: [PATCH 11/11] distributed prompt improvements --- profiles/claude.json | 12 ++++-------- profiles/gemini.json | 12 ++++-------- profiles/gpt.json | 12 ++++-------- profiles/llama.json | 12 ++++-------- 4 files changed, 16 insertions(+), 32 deletions(-) diff --git a/profiles/claude.json b/profiles/claude.json index a153363..f7402e0 100644 --- a/profiles/claude.json +++ b/profiles/claude.json @@ -59,10 +59,8 @@ ], [ - {"role": "user", "content": "hanky: collect some sand and give it to me please"}, - {"role": "assistant", "content": "Collecting sand... !newAction"}, - {"role": "system", "content": "Code Output:\nYou have reached player hanky.\nCode execution finished successfully."}, - {"role": "assistant", "content": "Here!"} + {"role": "user", "content": "hanky: collect some cobblestone"}, + {"role": "assistant", "content": "Sure, I can do that by collecting stone. !collectBlocks('stone', 10)"} ], [ @@ -87,10 +85,8 @@ ], [ - {"role": "user", "content": "maya: go to the nearest oak log"}, - {"role": "assistant", "content": "!newAction`"}, - {"role": "system", "content": "Code output:\n!!Code threw exception!! Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."}, - {"role": "assistant", "content": "Apologies, I made an error. Would you like me to try again?"} + {"role": "system", "content": "Code output: Action failed."}, + {"role": "assistant", "content": "Apologies, I made an error. Let me try again. !newAction"} ], [ diff --git a/profiles/gemini.json b/profiles/gemini.json index 09e6fe2..6ec033a 100644 --- a/profiles/gemini.json +++ b/profiles/gemini.json @@ -59,10 +59,8 @@ ], [ - {"role": "user", "content": "hanky: collect some sand and give it to me please"}, - {"role": "assistant", "content": "Collecting sand... !newAction"}, - {"role": "system", "content": "Code Output:\nYou have reached player hanky.\nCode execution finished successfully."}, - {"role": "assistant", "content": "Here!"} + {"role": "user", "content": "hanky: collect some cobblestone"}, + {"role": "assistant", "content": "Sure, I can do that by collecting stone. !collectBlocks('stone', 10)"} ], [ @@ -87,10 +85,8 @@ ], [ - {"role": "user", "content": "maya: go to the nearest oak log"}, - {"role": "assistant", "content": "!newAction`"}, - {"role": "system", "content": "Code output:\n!!Code threw exception!! Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."}, - {"role": "assistant", "content": "Apologies, I made an error. Would you like me to try again?"} + {"role": "system", "content": "Code output: Action failed."}, + {"role": "assistant", "content": "Apologies, I made an error. Let me try again. !newAction"} ], [ diff --git a/profiles/gpt.json b/profiles/gpt.json index a92aa88..a679cf8 100644 --- a/profiles/gpt.json +++ b/profiles/gpt.json @@ -59,10 +59,8 @@ ], [ - {"role": "user", "content": "hanky: collect some sand and give it to me please"}, - {"role": "assistant", "content": "Collecting sand... !newAction"}, - {"role": "system", "content": "Code Output:\nYou have reached player hanky.\nCode execution finished successfully."}, - {"role": "assistant", "content": "Here!"} + {"role": "user", "content": "hanky: collect some cobblestone"}, + {"role": "assistant", "content": "Sure, I can do that by collecting stone. !collectBlocks('stone', 10)"} ], [ @@ -87,10 +85,8 @@ ], [ - {"role": "user", "content": "maya: go to the nearest oak log"}, - {"role": "assistant", "content": "!newAction`"}, - {"role": "system", "content": "Code output:\n!!Code threw exception!! Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."}, - {"role": "assistant", "content": "Apologies, I made an error. Would you like me to try again?"} + {"role": "system", "content": "Code output: Action failed."}, + {"role": "assistant", "content": "Apologies, I made an error. Let me try again. !newAction"} ], [ diff --git a/profiles/llama.json b/profiles/llama.json index 63db4e5..b4d2dab 100644 --- a/profiles/llama.json +++ b/profiles/llama.json @@ -59,10 +59,8 @@ ], [ - {"role": "user", "content": "hanky: collect some sand and give it to me please"}, - {"role": "assistant", "content": "Collecting sand... !newAction"}, - {"role": "system", "content": "Code Output:\nYou have reached player hanky.\nCode execution finished successfully."}, - {"role": "assistant", "content": "Here!"} + {"role": "user", "content": "hanky: collect some cobblestone"}, + {"role": "assistant", "content": "Sure, I can do that by collecting stone. !collectBlocks('stone', 10)"} ], [ @@ -87,10 +85,8 @@ ], [ - {"role": "user", "content": "maya: go to the nearest oak log"}, - {"role": "assistant", "content": "!newAction`"}, - {"role": "system", "content": "Code output:\n!!Code threw exception!! Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."}, - {"role": "assistant", "content": "Apologies, I made an error. Would you like me to try again?"} + {"role": "system", "content": "Code output: Action failed."}, + {"role": "assistant", "content": "Apologies, I made an error. Let me try again. !newAction"} ], [