From f3b379addfb33a3b6b1b9634c4419495e85fe68f Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Tue, 2 Apr 2024 19:18:13 -0500 Subject: [PATCH 1/7] remove block in way, include inventory in system --- src/agent/library/skills.js | 30 ++++++++++++++++++++---------- src/agent/prompter.js | 6 +++++- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index 106cd4d..2ed1c67 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -491,13 +491,28 @@ export async function placeBlock(bot, blockType, x, y, z) { * await skills.placeBlock(bot, "oak_log", position.x + 1, position.y - 1, position.x); **/ console.log('placing block...') - const target_dest = new Vec3(Math.floor(x), Math.floor(y), Math.floor(z)); - const empty_blocks = ['air', 'water', 'lava', 'grass', 'tall_grass', 'snow', 'dead_bush', 'fern']; - const targetBlock = bot.blockAt(target_dest); - if (!empty_blocks.includes(targetBlock.name)) { - log(bot, `Cannot place block at ${targetBlock.position} because ${targetBlock.name} is in the way.`); + let block = bot.inventory.items().find(item => item.name === blockType); + if (!block) { + log(bot, `Don't have any ${blockType} to place.`); return false; } + + const target_dest = new Vec3(Math.floor(x), Math.floor(y), Math.floor(z)); + const targetBlock = bot.blockAt(target_dest); + if (targetBlock.name === blockType) { + log(bot, `${blockType} already at ${targetBlock.position}.`); + return false; + } + const empty_blocks = ['air', 'water', 'lava', 'grass', 'tall_grass', 'snow', 'dead_bush', 'fern']; + if (!empty_blocks.includes(targetBlock.name)) { + log(bot, `${blockType} in the way at ${targetBlock.position}.`); + const removed = await breakBlockAt(bot, x, y, z); + if (!removed) { + log(bot, `Cannot place ${blockType} at ${targetBlock.position}: block in the way.`); + return false; + } + await new Promise(resolve => setTimeout(resolve, 200)); // wait for block to break + } // get the buildoffblock and facevec based on whichever adjacent block is not empty let buildOffBlock = null; let faceVec = null; @@ -515,11 +530,6 @@ export async function placeBlock(bot, blockType, x, y, z) { return false; } - let block = bot.inventory.items().find(item => item.name === blockType); - if (!block) { - log(bot, `Don't have any ${blockType} to place.`); - return false; - } const pos = bot.entity.position; const pos_above = pos.plus(Vec3(0,1,0)); const dont_move_for = ['torch', 'redstone_torch', 'redstone', 'lever', 'button', 'rail', 'detector_rail', 'powered_rail', 'activator_rail', 'tripwire_hook', 'tripwire', 'water_bucket']; diff --git a/src/agent/prompter.js b/src/agent/prompter.js index 0794247..d26f7a2 100644 --- a/src/agent/prompter.js +++ b/src/agent/prompter.js @@ -54,6 +54,10 @@ export class Prompter { let stats = await getCommand('!stats').perform(this.agent); prompt = prompt.replaceAll('$STATS', stats); } + if (prompt.includes('$INVENTORY')) { + let inventory = await getCommand('!inventory').perform(this.agent); + prompt = prompt.replaceAll('$INVENTORY', inventory); + } if (prompt.includes('$COMMAND_DOCS')) prompt = prompt.replaceAll('$COMMAND_DOCS', getCommandDocs()); if (prompt.includes('$CODE_DOCS')) @@ -68,7 +72,7 @@ export class Prompter { // check if there are any remaining placeholders with syntax $ let remaining = prompt.match(/\$[A-Z_]+/g); if (remaining !== null) { - console.warn('Unknown prompt placeholders:', remaining); + console.warn('Unknown prompt placeholders:', remaining.join(', ')); } return prompt; } From 2a3cc01c1cca048f0b3a64735e4c72189b520ee1 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Tue, 2 Apr 2024 19:18:49 -0500 Subject: [PATCH 2/7] add inventory to system prompt --- andy.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/andy.json b/andy.json index fbee7d3..f1584f5 100644 --- a/andy.json +++ b/andy.json @@ -3,9 +3,9 @@ "model": "gpt-3.5-turbo", - "conversing": "You are a playful Minecraft bot named $NAME that can converse with players, see, move, mine, build, and interact with the world by using commands. Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'Sure, I've stopped.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer('playername', 3)'. This is extremely important to me, take a deep breath and have fun :)\n$STATS\n$COMMAND_DOCS\n$EXAMPLES\nConversation Begin:", + "conversing": "You are a playful Minecraft bot named $NAME that can converse with players, see, move, mine, build, and interact with the world by using commands. Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'Sure, I've stopped.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer('playername', 3)'. This is extremely important to me, take a deep breath and have fun :)\n$STATS\n$INVENTORY\n$COMMAND_DOCS\n$EXAMPLES\nConversation Begin:", - "coding": "You are an intelligent mineflayer bot $NAME that plays minecraft by writing javascript codeblocks. Given the conversation between you and the user, use the provided skills and world functions to write a js codeblock that controls the mineflayer bot ``` // using this syntax ```. The code will be executed and you will recieve it's output. If you are satisfied with the response, respond without a codeblock in a conversational way. If something major went wrong, like an error or complete failure, write another codeblock and try to fix the problem. Minor mistakes are acceptable. Be maximally efficient, creative, and clear. Do not use commands !likeThis, only use codeblocks. Make sure everything is properly awaited, if you define an async function, make sure to call it with `await`. Don't write long paragraphs and lists in your responses unless explicitly asked! Only summarize the code you write with a sentence or two when done. This is extremely important to me, take a deep breath and good luck! \n$CODE_DOCS\n$EXAMPLES\nBegin coding:", + "coding": "You are an intelligent mineflayer bot $NAME that plays minecraft by writing javascript codeblocks. Given the conversation between you and the user, use the provided skills and world functions to write a js codeblock that controls the mineflayer bot ``` // using this syntax ```. The code will be executed and you will recieve it's output. If you are satisfied with the response, respond without a codeblock in a conversational way. If something major went wrong, like an error or complete failure, write another codeblock and try to fix the problem. Minor mistakes are acceptable. Be maximally efficient, creative, and clear. Do not use commands !likeThis, only use codeblocks. Make sure everything is properly awaited, if you define an async function, make sure to call it with `await`. Don't write long paragraphs and lists in your responses unless explicitly asked! Only summarize the code you write with a sentence or two when done. This is extremely important to me, take a deep breath and good luck! \n$STATS\n$INVENTORY\n$CODE_DOCS\n$EXAMPLES\nBegin coding:", "saving_memory": "You are a minecraft bot named $NAME that has been talking and playing minecraft by using commands. Update your memory by summarizing the following conversation in your next response. Store information that will help you improve as a Minecraft bot. Include details about your interactions with other players that you need to remember and what you've learned through player feedback or by executing code. Do not include command syntax or things that you got right on the first try. Be extremely brief and use as few words as possible.\nOld Memory: '$MEMORY'\nRecent conversation: \n$TO_SUMMARIZE\nSummarize your old memory and recent conversation into a new memory, and respond only with the memory text: ", From 8d37dc12d21bdb13fd521e9db9b621b1240c42c9 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Tue, 2 Apr 2024 19:19:30 -0500 Subject: [PATCH 3/7] chat on spawn, not login --- src/agent/agent.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index 669caa5..789a8c4 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -22,8 +22,8 @@ export class Agent { initModes(this); - this.bot.on('login', async () => { - console.log(`${this.name} logged in.`); + this.bot.once('spawn', async () => { + console.log(`${this.name} spawned.`); this.coder.clear(); const ignore_messages = [ From d82e3401ac61cfb8baa58b9caaedb82350f8f970 Mon Sep 17 00:00:00 2001 From: Torsten Homberger Date: Fri, 5 Apr 2024 01:59:45 +0200 Subject: [PATCH 4/7] Fix typo --- src/models/claude.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/claude.js b/src/models/claude.js index c98127d..7a03855 100644 --- a/src/models/claude.js +++ b/src/models/claude.js @@ -29,7 +29,7 @@ export class Claude { msg.role = 'user'; msg.content = 'SYSTEM: ' + msg.content; } - if (msg.role === prev_role && msg.role === 'assitant') { + if (msg.role === prev_role && msg.role === 'assistant') { // insert empty user message to separate assistant messages messages.push(filler); messages.push(msg); From a620971ed29594e287ab26d3528ba8555cedba93 Mon Sep 17 00:00:00 2001 From: Maximus Date: Fri, 5 Apr 2024 16:58:08 -0500 Subject: [PATCH 5/7] coding interrupt, small fixes --- src/agent/coder.js | 5 ++++- src/agent/commands/index.js | 2 +- src/agent/library/skills.js | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/agent/coder.js b/src/agent/coder.js index 815ff39..0551032 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -94,11 +94,14 @@ export class Coder { let code_return = null; let failures = 0; + const interrupt_return = {success: true, message: null, interrupted: true, timedout: false}; for (let i=0; i<5; i++) { if (this.agent.bot.interrupt_code) - return {success: true, message: null, interrupted: true, timedout: false}; + return interrupt_return; console.log(messages) let res = await this.agent.prompter.promptCoding(messages); + if (this.agent.bot.interrupt_code) + return interrupt_return; let contains_code = res.indexOf('```') !== -1; if (!contains_code) { if (res.indexOf('!newAction') !== -1) { diff --git a/src/agent/commands/index.js b/src/agent/commands/index.js index 264e7b6..a71916b 100644 --- a/src/agent/commands/index.js +++ b/src/agent/commands/index.js @@ -95,7 +95,7 @@ export async function executeCommand(agent, message) { export function getCommandDocs() { let docs = `\n*COMMAND DOCS\n You can use the following commands to perform actions and get information about the world. Use the commands with the syntax: !commandName or !commandName("arg1", 1.2, ...) if the command takes arguments.\n - Do not use codeblocks. Only use one command in each response, trailing commands and comments will be ignored. Use these commands frequently in your responses!\n`; + Do not use codeblocks. Only use one command in each response, trailing commands and comments will be ignored.\n`; for (let command of commandList) { docs += command.name + ': ' + command.description + '\n'; if (command.params) { diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index 2ed1c67..829ccbe 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -503,7 +503,7 @@ export async function placeBlock(bot, blockType, x, y, z) { log(bot, `${blockType} already at ${targetBlock.position}.`); return false; } - const empty_blocks = ['air', 'water', 'lava', 'grass', 'tall_grass', 'snow', 'dead_bush', 'fern']; + const empty_blocks = ['air', 'water', 'lava', 'grass', 'short_grass', 'tall_grass', 'snow', 'dead_bush', 'fern']; if (!empty_blocks.includes(targetBlock.name)) { log(bot, `${blockType} in the way at ${targetBlock.position}.`); const removed = await breakBlockAt(bot, x, y, z); From 6a387e60e09c5a6e91abd8847b968203588334e0 Mon Sep 17 00:00:00 2001 From: Maximus Date: Fri, 5 Apr 2024 17:24:46 -0500 Subject: [PATCH 6/7] added vec3 package --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 8e9aeb0..ae200aa 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "mineflayer-pvp": "^1.3.2", "openai": "^4.4.0", "patch-package": "^8.0.0", + "vec3": "^0.1.10", "yargs": "^17.7.2" }, "scripts": { From 97e2bfbe88195f609acea2354c60a7b36239d77c Mon Sep 17 00:00:00 2001 From: Maximus Date: Fri, 5 Apr 2024 17:25:06 -0500 Subject: [PATCH 7/7] better readme --- README.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 17c5c02..d44ad91 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,16 @@ Add one of these environment variables: - `GEMINI_API_KEY` - `ANTHROPIC_API_KEY` (and optionally `OPENAI_API_KEY` for embeddings. not necessary, but without embeddings performance will suffer) +⭐[How do I add the API key as an environment variable?](https://phoenixnap.com/kb/windows-set-environment-variable)⭐ + + Clone/Download this repository Run `npm install` -Install the minecraft version specified in `settings.json`, currently supports up to 1.20.2 +Install the minecraft version specified in `settings.json`, currently supports up to 1.20.4 -## Run +## Running Locally Start a minecraft world and open it to LAN on localhost port `55916` @@ -33,17 +36,21 @@ Run `node main.js` You can configure the agent's name, model, and prompts in their profile like `andy.json`. +You can configure project details in `settings.json`. -You can configure project details in `settings.json`. Here is an example settings for connecting to a non-local server: + +## Online Servers +To connect to online servers your bot will need an official Microsoft/Minecraft account. You can use your own personal one, but will need another account if you want to connect with it. Here is an example settings for this: ``` { - "minecraft_version": "1.20.1", + "minecraft_version": "1.20.4", "host": "111.222.333.444", "port": 55920, "auth": "microsoft", "allow_insecure_coding": false } ``` +‼️Make sure your bot's name in the profile.json matches the account name! Otherwise the bot will spam talk to itself. ## Patches