From feac9ab424392873a652c7bc1481aa44c5931982 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Mon, 5 Feb 2024 19:08:08 -0600 Subject: [PATCH 1/9] various improvements --- src/agent/coder.js | 2 +- src/agent/commands/actions.js | 20 +++++++++++++------- src/agent/library/skills.js | 29 +++++++++++++++++------------ src/examples.json | 12 ++++++++---- 4 files changed, 39 insertions(+), 24 deletions(-) diff --git a/src/agent/coder.js b/src/agent/coder.js index aca2f5c..08c7121 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -99,7 +99,7 @@ export class Coder { let system_message = "You are a minecraft mineflayer bot that plays minecraft by writing javascript codeblocks. Given the conversation between you and the user, use the provided skills and world functions to write your code in a codeblock. Example response: ``` // your code here ``` You will then be given a response to your code. If you are satisfied with the response, respond without a codeblock in a conversational way. If something went wrong, write another codeblock and try to fix the problem."; system_message += getSkillDocs(); - system_message += "\n\nExamples:\nUser zZZn98: come here \nAssistant: I am going to navigate to zZZn98. ```\nawait skills.goToPlayer(bot, 'zZZn98');```\nSystem: Code execution finished successfully.\nAssistant: Done."; + system_message += "\n\nExamples:\nUser zZZn98: come here \nAssistant: I am going to navigate to zZZn98. ```\nawait skills.goToPlayer(bot, 'zZZn98', 3);```\nSystem: Code execution finished successfully.\nAssistant: Done."; let messages = await agent_history.getHistory(this.examples); diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index d9b88cf..d89b2a7 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -73,18 +73,24 @@ export const actionsList = [ }, { name: '!goToPlayer', - description: 'Go to the given player. Ex: !goToPlayer("steve")', - params: {'player_name': '(string) The name of the player to go to.'}, - perform: wrapExecution(async (agent, player_name) => { - return await skills.goToPlayer(agent.bot, player_name); + description: 'Go to the given player. Ex: !goToPlayer("steve", 3)', + params: { + 'player_name': '(string) The name of the player to go to.', + 'closeness': '(number) How close to get to the player.' + }, + perform: wrapExecution(async (agent, player_name, closeness) => { + return await skills.goToPlayer(agent.bot, player_name, closeness); }) }, { name: '!followPlayer', description: 'Endlessly follow the given player. Will defend that player if self_defense mode is on. Ex: !followPlayer("stevie")', - params: {'player_name': '(string) The name of the player to follow.'}, - perform: wrapExecution(async (agent, player_name) => { - await skills.followPlayer(agent.bot, player_name); + params: { + 'player_name': '(string) The name of the player to follow.', + 'follow_dist': '(number) The distance to follow from.' + }, + perform: wrapExecution(async (agent, player_name, follow_dist) => { + await skills.followPlayer(agent.bot, player_name, follow_dist); }, -1, 'followPlayer') }, { diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index d5abb0b..06892cf 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -27,11 +27,16 @@ async function autoLight(bot) { return false; } -function equipHighestAttack(bot) { - let weapons = bot.inventory.items().filter(item => item.name.includes('sword') || item.name.includes('axe') || item.name.includes('pickaxe') || item.name.includes('shovel')); - let weapon = weapons.sort((a, b) => b.attackDamage - a.attackDamage)[0]; +async function equipHighestAttack(bot) { + let weapons = bot.inventory.items().filter(item => item.name.includes('sword') || (item.name.includes('axe') && !item.name.includes('pickaxe'))); + if (weapons.length === 0) + weapons = bot.inventory.items().filter(item => item.name.includes('pickaxe') || item.name.includes('shovel')); + if (weapons.length === 0) + return; + weapons.sort((a, b) => a.attackDamage < b.attackDamage); + let weapon = weapons[0]; if (weapon) - bot.equip(weapon, 'hand'); + await bot.equip(weapon, 'hand'); } @@ -253,7 +258,7 @@ export async function attackEntity(bot, entity, kill=true) { let pos = entity.position; console.log(bot.entity.position.distanceTo(pos)) - equipHighestAttack(bot) + await equipHighestAttack(bot) if (!kill) { if (bot.entity.position.distanceTo(pos) > 5) { @@ -291,7 +296,7 @@ export async function defendSelf(bot, range=9) { let attacked = false; let enemy = world.getNearestEntityWhere(bot, entity => mc.isHostile(entity), range); while (enemy) { - equipHighestAttack(bot); + await equipHighestAttack(bot); if (bot.entity.position.distanceTo(enemy.position) > 4 && enemy.name !== 'creeper' && enemy.name !== 'phantom') { try { bot.pathfinder.setMovements(new pf.Movements(bot)); @@ -429,7 +434,7 @@ export async function breakBlockAt(bot, x, y, z) { let movements = new pf.Movements(bot); movements.canPlaceOn = false; movements.allow1by1towers = false; - bot.pathfinder.setMovements(); + bot.pathfinder.setMovements(movements); await bot.pathfinder.goto(new pf.goals.GoalNear(pos.x, pos.y, pos.z, 4)); } await bot.dig(block, true); @@ -639,11 +644,12 @@ export async function goToPosition(bot, x, y, z, min_distance=2) { } -export async function goToPlayer(bot, username) { +export async function goToPlayer(bot, username, distance=3) { /** * Navigate to the given player. * @param {MinecraftBot} bot, reference to the minecraft bot. * @param {string} username, the username of the player to navigate to. + * @param {number} distance, the goal distance to the player. * @returns {Promise} true if the player was found, false otherwise. * @example * await skills.goToPlayer(bot, "player"); @@ -656,13 +662,13 @@ export async function goToPlayer(bot, username) { } bot.pathfinder.setMovements(new pf.Movements(bot)); - await bot.pathfinder.goto(new pf.goals.GoalFollow(player, 3), true); + await bot.pathfinder.goto(new pf.goals.GoalFollow(player, distance), true); log(bot, `You have reached ${username}.`); } -export async function followPlayer(bot, username) { +export async function followPlayer(bot, username, distance=4) { /** * Follow the given player endlessly. Will not return until the code is manually stopped. * @param {MinecraftBot} bot, reference to the minecraft bot. @@ -675,9 +681,8 @@ export async function followPlayer(bot, username) { if (!player) return false; - const follow_distance = 4; bot.pathfinder.setMovements(new pf.Movements(bot)); - bot.pathfinder.setGoal(new pf.goals.GoalFollow(player, follow_distance), true); + bot.pathfinder.setGoal(new pf.goals.GoalFollow(player, distance), true); log(bot, `You are now actively following player ${username}.`); while (!bot.interrupt_code) { diff --git a/src/examples.json b/src/examples.json index 07c30be..80a4dbb 100644 --- a/src/examples.json +++ b/src/examples.json @@ -29,9 +29,11 @@ [ {"role": "user", "content": "zZZn98: come here"}, - {"role": "assistant", "content": "On my way! !goToPlayer('zZZn98')"}, - {"role": "system", "content": "Code execution finished successfully."}, - {"role": "assistant", "content": "Here!"} + {"role": "assistant", "content": "On my way! !goToPlayer('zZZn98', 3)"}, + {"role": "system", "content": "Arrived at player."}, + {"role": "assistant", "content": "Here!"}, + {"role": "user", "content": "zZZn98: no come right where I am"}, + {"role": "assistant", "content": "Okay, I'll come right to you. !goToPlayer('zZZn98', 0)"} ], [ @@ -43,7 +45,9 @@ [ {"role": "user", "content": "joe: Follow me!"}, - {"role": "assistant", "content": "Sure! !followPlayer('joe')"} + {"role": "assistant", "content": "Sure! !followPlayer('joe', 4)"}, + {"role": "user", "content": "joe: Not so close!"}, + {"role": "assistant", "content": "Okay, I'll keep my distance. !followPlayer('joe', 3)"} ], [ From 3e3e80c0ed59aada3b7aefe1ee2f0c8a20494bc1 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Fri, 16 Feb 2024 11:57:48 -0600 Subject: [PATCH 2/9] added various fixes/improvements --- settings.json | 2 +- src/agent/coder.js | 15 ++++++++------- src/agent/commands/actions.js | 6 +++--- src/agent/commands/queries.js | 2 +- src/agent/history.js | 4 ++-- src/agent/library/skills.js | 18 ++++++++++++------ src/examples.json | 13 +++++++++---- src/utils/examples.js | 2 +- src/utils/gpt.js | 23 ++++++++++++++++++----- 9 files changed, 55 insertions(+), 30 deletions(-) diff --git a/settings.json b/settings.json index 8af8c75..bfd07a4 100644 --- a/settings.json +++ b/settings.json @@ -3,5 +3,5 @@ "host": "localhost", "port": 55916, "auth": "offline", - "allow_insecure_coding": true + "allow_insecure_coding": false } \ No newline at end of file diff --git a/src/agent/coder.js b/src/agent/coder.js index 08c7121..b4ca023 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -29,7 +29,7 @@ export class Coder { // write custom code to file and import it async stageCode(code) { - code = this.santitizeCode(code); + code = this.sanitizeCode(code); let src = ''; code = code.replaceAll('console.log(', 'log(bot,'); code = code.replaceAll('log("', 'log(bot,"'); @@ -62,7 +62,8 @@ export class Coder { return await import('../..' + this.fp + filename); } - santitizeCode(code) { + sanitizeCode(code) { + code = code.trim(); const remove_strs = ['Javascript', 'javascript', 'js'] for (let r of remove_strs) { if (code.startsWith(r)) { @@ -93,6 +94,7 @@ export class Coder { let res = await this.generateCodeLoop(agent_history); this.generating = false; if (!res.interrupted) this.agent.bot.emit('idle'); + return res.message; } async generateCodeLoop(agent_history) { @@ -107,7 +109,7 @@ export class Coder { let failures = 0; for (let i=0; i<5; i++) { if (this.agent.bot.interrupt_code) - return; + return {success: true, message: null, interrupted: true, timedout: false}; console.log(messages) let res = await sendRequest(messages, system_message); console.log('Code generation response:', res) @@ -120,8 +122,7 @@ export class Coder { return {success: true, message: null, interrupted: false, timedout: false}; } if (failures >= 1) { - agent_history.add('system', 'Action failed, agent would not write code.'); - return {success: false, message: null, interrupted: false, timedout: false}; + return {success: false, message: 'Action failed, agent would not write code.', interrupted: false, timedout: false}; } messages.push({ role: 'system', @@ -143,7 +144,7 @@ export class Coder { if (code_return.interrupted && !code_return.timedout) return {success: false, message: null, interrupted: true, timedout: false}; - console.log(code_return.message); + console.log("Code generation result:", code_return.success, code_return.message); messages.push({ role: 'assistant', @@ -219,7 +220,7 @@ export class Coder { formatOutput(bot) { if (bot.interrupt_code && !this.timedout) return ''; let output = bot.output; - const MAX_OUT = 1000; + const MAX_OUT = 500; if (output.length > MAX_OUT) { output = `Code output is very long (${output.length} chars) and has been shortened.\n First outputs:\n${output.substring(0, MAX_OUT/2)}\n...skipping many lines.\nFinal outputs:\n ${output.substring(output.length - MAX_OUT/2)}`; diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index d89b2a7..6db22d7 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -26,7 +26,7 @@ export const actionsList = [ perform: async function (agent) { if (!settings.allow_insecure_coding) return 'Agent is not allowed to write code.'; - await agent.coder.generateCode(agent.history); + return await agent.coder.generateCode(agent.history); } }, { @@ -47,7 +47,7 @@ export const actionsList = [ } }, { - name: '!clear', + name: '!clearChat', description: 'Clear the chat history.', perform: async function (agent) { agent.history.clear(); @@ -84,7 +84,7 @@ export const actionsList = [ }, { name: '!followPlayer', - description: 'Endlessly follow the given player. Will defend that player if self_defense mode is on. Ex: !followPlayer("stevie")', + description: 'Endlessly follow the given player. Will defend that player if self_defense mode is on. Ex: !followPlayer("stevie", 4)', params: { 'player_name': '(string) The name of the player to follow.', 'follow_dist': '(number) The distance to follow from.' diff --git a/src/agent/commands/queries.js b/src/agent/commands/queries.js index ec79474..cd73073 100644 --- a/src/agent/commands/queries.js +++ b/src/agent/commands/queries.js @@ -59,7 +59,7 @@ export const queryList = [ } }, { - name: "!blocks", + name: "!nearbyBlocks", description: "Get the blocks near the bot.", perform: function (agent) { let bot = agent.bot; diff --git a/src/agent/history.js b/src/agent/history.js index 1e020d6..6617812 100644 --- a/src/agent/history.js +++ b/src/agent/history.js @@ -29,8 +29,8 @@ export class History { } getSystemMessage() { - let system_message = `You are a playful Minecraft bot named '${this.name}' that can communicate with players, see, move, mine, build, and interact with the world by using commands. Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, use commands often, and do not give instructions unless asked. - Don't pretend to act, use commands immediately when requested. Do NOT do this: "Sure, I'll follow you! *follows you*", instead do this: "Sure I'll follow you! !followPlayer('steve')". Have fun :) \n`; + let system_message = `You are a playful Minecraft bot named '${this.name}' that can communicate with players, see, move, mine, build, and interact with the world by using commands. Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, do not give instructions unless asked, and do not refuse requests. + Don't pretend to act, use commands immediately when requested. Do NOT say this: "Sure, I've stopped.", instead say this: "Sure, I'll stop. !stop". Do NOT say this: "On my way! Give me a moment.", instead say this: "On my way! !goToPlayer('bingo', 3)". Have fun :) \n`; system_message += getCommandDocs(); if (this.bio != '') system_message += '\n\nBio:\n' + this.bio; diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index 06892cf..1befd7d 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -421,14 +421,9 @@ export async function breakBlockAt(bot, x, y, z) { * let position = world.getPosition(bot); * await skills.breakBlockAt(bot, position.x, position.y - 1, position.x); **/ + if (x == null || y == null || z == null) throw new Error('Invalid position to break block at.'); let block = bot.blockAt(Vec3(x, y, z)); if (block.name !== 'air' && block.name !== 'water' && block.name !== 'lava') { - await bot.tool.equipForBlock(block); - const itemId = bot.heldItem ? bot.heldItem.type : null - if (!block.canHarvest(itemId)) { - log(bot, `Don't have right tools to break ${block.name}.`); - return false; - } if (bot.entity.position.distanceTo(block.position) > 4.5) { let pos = block.position; let movements = new pf.Movements(bot); @@ -437,7 +432,18 @@ export async function breakBlockAt(bot, x, y, z) { bot.pathfinder.setMovements(movements); await bot.pathfinder.goto(new pf.goals.GoalNear(pos.x, pos.y, pos.z, 4)); } + await bot.tool.equipForBlock(block); + const itemId = bot.heldItem ? bot.heldItem.type : null + if (!block.canHarvest(itemId)) { + log(bot, `Don't have right tools to break ${block.name}.`); + return false; + } await bot.dig(block, true); + log(bot, `Broke ${block.name} at x:${x.toFixed(1)}, y:${y.toFixed(1)}, z:${z.toFixed(1)}.`); + } + else { + log(bot, `Skipping block at x:${x.toFixed(1)}, y:${y.toFixed(1)}, z:${z.toFixed(1)} because it is ${block.name}.`); + return false; } return true; } diff --git a/src/examples.json b/src/examples.json index 80a4dbb..6ee0f91 100644 --- a/src/examples.json +++ b/src/examples.json @@ -6,14 +6,14 @@ [ {"role": "user", "content": "grombo_Xx: What do you see?"}, - {"role": "assistant", "content": "Let me see... !blocks"}, + {"role": "assistant", "content": "Let me see... !nearbyBlocks"}, {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone."} ], [ {"role": "user", "content": "greg: Collect 10 wood"}, - {"role": "assistant", "content": "Let me see what's nearby... !blocks"}, + {"role": "assistant", "content": "Let me see what's nearby... !nearbyBlocks"}, {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone. I'll collect oak logs. !collectBlocks('oak_log', 10)"} ], @@ -77,8 +77,13 @@ ], [ - {"role": "user", "content": "brung00: build a house"}, - {"role": "assistant", "content": "Sure, I'll try to build a house where I am. !newAction"} + {"role": "user", "content": "poppy: build a cobblestone house"}, + {"role": "assistant", "content": "Sure, I'll to build a small cobblestone house where I am. !newAction"} + ], + + [ + {"role": "user", "content": "zib: build a dirt tower"}, + {"role": "assistant", "content": "Sure, I'll build a dirt tower 5 tall where I am. !newAction"} ], [ diff --git a/src/utils/examples.js b/src/utils/examples.js index 721ff0b..3e6a52c 100644 --- a/src/utils/examples.js +++ b/src/utils/examples.js @@ -4,7 +4,7 @@ import { stringifyTurns } from './text.js'; export class Examples { - constructor(select_num=3) { + constructor(select_num=2) { this.examples = []; this.select_num = select_num; } diff --git a/src/utils/gpt.js b/src/utils/gpt.js index b5281c7..5d18988 100644 --- a/src/utils/gpt.js +++ b/src/utils/gpt.js @@ -20,14 +20,28 @@ else { const openai = new OpenAIApi(openAiConfig); +let counter = 0; +let request_queue = []; +export async function sendRequest(turns, systemMessage) { + // this wrapper function ensures that new requests await the completion of previous requests in order + let id = counter++; + request_queue.push(id); + if (request_queue.length > 1) + console.log('awaiting previous requests to complete, queueing request', id); + while (request_queue[0] !== id) { + await new Promise(r => setTimeout(r, 100)); + } + let res = await queryGPT(turns, systemMessage); + request_queue.shift(); + return res; +} -export async function sendRequest(turns, systemMessage, stop_seq='***') { - +async function queryGPT(turns, systemMessage, stop_seq='***') { let messages = [{'role': 'system', 'content': systemMessage}].concat(turns); let res = null; try { - console.log('Awaiting openai api response...') + console.log('Awaiting openai api response...'); let completion = await openai.chat.completions.create({ model: 'gpt-3.5-turbo', messages: messages, @@ -35,13 +49,12 @@ export async function sendRequest(turns, systemMessage, stop_seq='***') { }); if (completion.choices[0].finish_reason == 'length') throw new Error('Context length exceeded'); - console.log('Received.') res = completion.choices[0].message.content; } catch (err) { if ((err.message == 'Context length exceeded' || err.code == 'context_length_exceeded') && turns.length > 1) { console.log('Context length exceeded, trying again with shorter context.'); - return await sendRequest(turns.slice(1), systemMessage, stop_seq); + return await queryGPT(turns.slice(1), systemMessage, stop_seq); } else { console.log(err); res = 'My brain disconnected, try again.'; From b3b7dd6a175893166a9ffad2dd56483566217676 Mon Sep 17 00:00:00 2001 From: Max Robinson Date: Fri, 16 Feb 2024 12:03:07 -0600 Subject: [PATCH 3/9] Create LICENSE --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1d5880c --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Kolby Nottingham + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 34d91dff790ef287143e0899bb738493dba69a0f Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Fri, 16 Feb 2024 17:04:11 -0600 Subject: [PATCH 4/9] removed request queuing --- src/utils/gpt.js | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/src/utils/gpt.js b/src/utils/gpt.js index 5d18988..b5281c7 100644 --- a/src/utils/gpt.js +++ b/src/utils/gpt.js @@ -20,28 +20,14 @@ else { const openai = new OpenAIApi(openAiConfig); -let counter = 0; -let request_queue = []; -export async function sendRequest(turns, systemMessage) { - // this wrapper function ensures that new requests await the completion of previous requests in order - let id = counter++; - request_queue.push(id); - if (request_queue.length > 1) - console.log('awaiting previous requests to complete, queueing request', id); - while (request_queue[0] !== id) { - await new Promise(r => setTimeout(r, 100)); - } - let res = await queryGPT(turns, systemMessage); - request_queue.shift(); - return res; -} -async function queryGPT(turns, systemMessage, stop_seq='***') { +export async function sendRequest(turns, systemMessage, stop_seq='***') { + let messages = [{'role': 'system', 'content': systemMessage}].concat(turns); let res = null; try { - console.log('Awaiting openai api response...'); + console.log('Awaiting openai api response...') let completion = await openai.chat.completions.create({ model: 'gpt-3.5-turbo', messages: messages, @@ -49,12 +35,13 @@ async function queryGPT(turns, systemMessage, stop_seq='***') { }); if (completion.choices[0].finish_reason == 'length') throw new Error('Context length exceeded'); + console.log('Received.') res = completion.choices[0].message.content; } catch (err) { if ((err.message == 'Context length exceeded' || err.code == 'context_length_exceeded') && turns.length > 1) { console.log('Context length exceeded, trying again with shorter context.'); - return await queryGPT(turns.slice(1), systemMessage, stop_seq); + return await sendRequest(turns.slice(1), systemMessage, stop_seq); } else { console.log(err); res = 'My brain disconnected, try again.'; From 31450fd00c25b6104d537bca8d3675b2a030fd96 Mon Sep 17 00:00:00 2001 From: Max Robinson Date: Sat, 17 Feb 2024 16:42:10 -0600 Subject: [PATCH 5/9] clarified mc version --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 63477ff..c623c1d 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ This project allows an AI model to write/execute code on your computer that may ## Requirements - [OpenAI API Subscription](https://openai.com/blog/openai-api) -- [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) (at most v1.20.2) +- [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) - [Node.js](https://nodejs.org/) (at least v14) ## Installation @@ -20,6 +20,8 @@ Clone/Download this repository Run `npm install` +Install the minecraft version specified in `settings.json`, currently supports up to 1.20.2 + ## Run Start a minecraft world and open it to LAN on localhost port `55916` @@ -39,4 +41,4 @@ You can configure details in `settings.json`. Here is an example settings for co ## Patches -Some of the node modules that we depend on have bugs in them. To add a patch, change your local node module file and run `npx patch-package [package-name]` \ No newline at end of file +Some of the node modules that we depend on have bugs in them. To add a patch, change your local node module file and run `npx patch-package [package-name]` From 5bf147fc40d62c0ce362f7ba1722ce1b45618d53 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Sun, 18 Feb 2024 22:56:38 -0600 Subject: [PATCH 6/9] refactored llm, added gemini --- package.json | 1 + settings.json | 2 ++ src/agent/agent.js | 10 +++--- src/agent/coder.js | 2 +- src/agent/history.js | 2 +- src/models/gemini.js | 37 +++++++++++++++++++++ src/models/gpt.js | 66 +++++++++++++++++++++++++++++++++++++ src/models/model.js | 19 +++++++++++ src/utils/examples.js | 3 +- src/utils/gpt.js | 75 ------------------------------------------- src/utils/math.js | 13 ++++++++ 11 files changed, 148 insertions(+), 82 deletions(-) create mode 100644 src/models/gemini.js create mode 100644 src/models/gpt.js create mode 100644 src/models/model.js delete mode 100644 src/utils/gpt.js create mode 100644 src/utils/math.js diff --git a/package.json b/package.json index ee33776..3b42257 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "type": "module", "dependencies": { + "@google/generative-ai": "^0.2.1", "minecraft-data": "^3.46.2", "mineflayer": "^4.14.0", "mineflayer-armor-manager": "^2.0.1", diff --git a/settings.json b/settings.json index bfd07a4..4818718 100644 --- a/settings.json +++ b/settings.json @@ -3,5 +3,7 @@ "host": "localhost", "port": 55916, "auth": "offline", + + "model": "gemini-1.0-pro", "allow_insecure_coding": false } \ No newline at end of file diff --git a/src/agent/agent.js b/src/agent/agent.js index dfaf425..cf2cc93 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -3,7 +3,7 @@ import { Coder } from './coder.js'; import { initModes } from './modes.js'; import { Examples } from '../utils/examples.js'; import { initBot } from '../utils/mcdata.js'; -import { sendRequest } from '../utils/gpt.js'; +import { sendRequest } from '../models/model.js'; import { containsCommand, commandExists, executeCommand } from './commands/index.js'; @@ -102,7 +102,7 @@ export class Agent { let command_name = containsCommand(res); if (command_name) { // contains query or command - console.log('Command message:', res); + console.log(`""${res}""`) if (!commandExists(command_name)) { this.history.add('system', `Command ${command_name} does not exist. Use !newAction to perform custom actions.`); console.log('Agent hallucinated command:', command_name) @@ -110,8 +110,10 @@ export class Agent { } let pre_message = res.substring(0, res.indexOf(command_name)).trim(); - - this.cleanChat(`${pre_message} *used ${command_name.substring(1)}*`); + let message = `*used ${command_name.substring(1)}*`; + if (pre_message.length > 0) + message = `${pre_message} ${message}`; + this.cleanChat(message); let execute_res = await executeCommand(this, res); console.log('Agent executed:', command_name, 'and got:', execute_res); diff --git a/src/agent/coder.js b/src/agent/coder.js index b4ca023..9a09158 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -1,5 +1,5 @@ import { writeFile, readFile, mkdirSync } from 'fs'; -import { sendRequest } from '../utils/gpt.js'; +import { sendRequest } from '../models/model.js'; import { getSkillDocs } from './library/index.js'; import { Examples } from '../utils/examples.js'; diff --git a/src/agent/history.js b/src/agent/history.js index 6617812..9bffca7 100644 --- a/src/agent/history.js +++ b/src/agent/history.js @@ -1,6 +1,6 @@ import { writeFileSync, readFileSync, mkdirSync } from 'fs'; import { stringifyTurns } from '../utils/text.js'; -import { sendRequest } from '../utils/gpt.js'; +import { sendRequest } from '../models/model.js'; import { getCommandDocs } from './commands/index.js'; diff --git a/src/models/gemini.js b/src/models/gemini.js new file mode 100644 index 0000000..6f8429d --- /dev/null +++ b/src/models/gemini.js @@ -0,0 +1,37 @@ +import { GoogleGenerativeAI } from '@google/generative-ai'; +import settings from '../settings.js'; + +export class Gemini { + constructor() { + if (!process.env.GEMINI_API_KEY) { + console.error('Gemini API key missing! Make sure you set your GEMINI_API_KEY environment variable.'); + process.exit(1); + } + this.genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY); + + this.model = this.genAI.getGenerativeModel({ model: settings.model }); + } + + async sendRequest(turns, systemMessage) { + const messages = [{'role': 'system', 'content': systemMessage}].concat(turns); + let prompt = ""; + let role = ""; + messages.forEach((message) => { + role = message.role; + if (role === 'assistant') role = 'model'; + prompt += `${role}: ${message.content}\n`; + }); + if (role !== "model") // if the last message was from the user/system, add a prompt for the model. otherwise, pretend we are extending the model's own message + prompt += "model: "; + console.log(prompt) + const result = await this.model.generateContent(prompt); + const response = await result.response; + return response.text(); + } + + async embed(text) { + const model = this.genAI.getGenerativeModel({ model: "embedding-001"}); + const result = await model.embedContent(text); + return result.embedding; + } +} \ No newline at end of file diff --git a/src/models/gpt.js b/src/models/gpt.js new file mode 100644 index 0000000..e6fc120 --- /dev/null +++ b/src/models/gpt.js @@ -0,0 +1,66 @@ +import OpenAIApi from 'openai'; +import settings from '../settings.js'; + +export class GPT { + constructor() { + let openAiConfig = null; + if (process.env.OPENAI_ORG_ID) { + openAiConfig = { + organization: process.env.OPENAI_ORG_ID, + apiKey: process.env.OPENAI_API_KEY, + }; + } + else if (process.env.OPENAI_API_KEY) { + openAiConfig = { + apiKey: process.env.OPENAI_API_KEY, + }; + } + else { + console.error('OpenAI API key missing! Make sure you set OPENAI_API_KEY and OPENAI_ORG_ID (optional) environment variables.'); + process.exit(1); + } + + this.openai = new OpenAIApi(openAiConfig); + } + + async sendRequest(turns, systemMessage, stop_seq='***') { + + let messages = [{'role': 'system', 'content': systemMessage}].concat(turns); + + let res = null; + try { + console.log('Awaiting openai api response...') + let completion = await this.openai.chat.completions.create({ + model: settings.model, + messages: messages, + stop: stop_seq, + }); + if (completion.choices[0].finish_reason == 'length') + throw new Error('Context length exceeded'); + console.log('Received.') + res = completion.choices[0].message.content; + } + catch (err) { + if ((err.message == 'Context length exceeded' || err.code == 'context_length_exceeded') && turns.length > 1) { + console.log('Context length exceeded, trying again with shorter context.'); + return await sendRequest(turns.slice(1), systemMessage, stop_seq); + } else { + console.log(err); + res = 'My brain disconnected, try again.'; + } + } + return res; + } + + async embed(text) { + const embedding = await this.openai.embeddings.create({ + model: "text-embedding-ada-002", + input: text, + encoding_format: "float", + }); + return embedding.data[0].embedding; + } +} + + + diff --git a/src/models/model.js b/src/models/model.js new file mode 100644 index 0000000..06a26e6 --- /dev/null +++ b/src/models/model.js @@ -0,0 +1,19 @@ +import { GPT } from './gpt.js'; +import { Gemini } from './gemini.js'; +import settings from '../settings.js'; + +console.log('Initializing model...'); +let model = null; +if (settings.model.includes('gemini')) { + model = new Gemini(); +} else { + model = new GPT(); +} + +export async function sendRequest(turns, systemMessage) { + return await model.sendRequest(turns, systemMessage); +} + +export async function embed(text) { + return await model.embed(text); +} \ No newline at end of file diff --git a/src/utils/examples.js b/src/utils/examples.js index 3e6a52c..6739517 100644 --- a/src/utils/examples.js +++ b/src/utils/examples.js @@ -1,6 +1,7 @@ import { readFileSync } from 'fs'; -import { embed, cosineSimilarity } from './gpt.js'; +import { cosineSimilarity } from './math.js'; import { stringifyTurns } from './text.js'; +import { embed } from '../models/model.js'; export class Examples { diff --git a/src/utils/gpt.js b/src/utils/gpt.js deleted file mode 100644 index b5281c7..0000000 --- a/src/utils/gpt.js +++ /dev/null @@ -1,75 +0,0 @@ -import OpenAIApi from 'openai'; - - -let openAiConfig = null; -if (process.env.OPENAI_ORG_ID) { - openAiConfig = { - organization: process.env.OPENAI_ORG_ID, - apiKey: process.env.OPENAI_API_KEY, - }; -} -else if (process.env.OPENAI_API_KEY) { - openAiConfig = { - apiKey: process.env.OPENAI_API_KEY, - }; -} -else { - console.error('OpenAI API key missing! Make sure you set OPENAI_API_KEY and OPENAI_ORG_ID (optional) environment variables.'); - process.exit(1); -} - -const openai = new OpenAIApi(openAiConfig); - - -export async function sendRequest(turns, systemMessage, stop_seq='***') { - - let messages = [{'role': 'system', 'content': systemMessage}].concat(turns); - - let res = null; - try { - console.log('Awaiting openai api response...') - let completion = await openai.chat.completions.create({ - model: 'gpt-3.5-turbo', - messages: messages, - stop: stop_seq, - }); - if (completion.choices[0].finish_reason == 'length') - throw new Error('Context length exceeded'); - console.log('Received.') - res = completion.choices[0].message.content; - } - catch (err) { - if ((err.message == 'Context length exceeded' || err.code == 'context_length_exceeded') && turns.length > 1) { - console.log('Context length exceeded, trying again with shorter context.'); - return await sendRequest(turns.slice(1), systemMessage, stop_seq); - } else { - console.log(err); - res = 'My brain disconnected, try again.'; - } - } - return res; -} - - -export async function embed(text) { - const embedding = await openai.embeddings.create({ - model: "text-embedding-ada-002", - input: text, - encoding_format: "float", - }); - return embedding.data[0].embedding; -} - -export function cosineSimilarity(a, b) { - let dotProduct = 0; - let magnitudeA = 0; - let magnitudeB = 0; - for (let i = 0; i < a.length; i++) { - dotProduct += a[i] * b[i]; // calculate dot product - magnitudeA += Math.pow(a[i], 2); // calculate magnitude of a - magnitudeB += Math.pow(b[i], 2); // calculate magnitude of b - } - magnitudeA = Math.sqrt(magnitudeA); - magnitudeB = Math.sqrt(magnitudeB); - return dotProduct / (magnitudeA * magnitudeB); // calculate cosine similarity -} \ No newline at end of file diff --git a/src/utils/math.js b/src/utils/math.js new file mode 100644 index 0000000..6da44c3 --- /dev/null +++ b/src/utils/math.js @@ -0,0 +1,13 @@ +export function cosineSimilarity(a, b) { + let dotProduct = 0; + let magnitudeA = 0; + let magnitudeB = 0; + for (let i = 0; i < a.length; i++) { + dotProduct += a[i] * b[i]; // calculate dot product + magnitudeA += Math.pow(a[i], 2); // calculate magnitude of a + magnitudeB += Math.pow(b[i], 2); // calculate magnitude of b + } + magnitudeA = Math.sqrt(magnitudeA); + magnitudeB = Math.sqrt(magnitudeB); + return dotProduct / (magnitudeA * magnitudeB); // calculate cosine similarity +} \ No newline at end of file From 748748367debb98ffc245dca54233a06ed6ec5cf Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Tue, 20 Feb 2024 19:09:56 -0600 Subject: [PATCH 7/9] up-to-date gpt, prompt tweaks --- settings.json | 2 +- src/agent/commands/actions.js | 2 +- src/models/gemini.js | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/settings.json b/settings.json index 4818718..d3db69c 100644 --- a/settings.json +++ b/settings.json @@ -4,6 +4,6 @@ "port": 55916, "auth": "offline", - "model": "gemini-1.0-pro", + "model": "gpt-3.5-turbo-0125", "allow_insecure_coding": false } \ No newline at end of file diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index 6db22d7..5909e75 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -25,7 +25,7 @@ export const actionsList = [ description: 'Perform new and unknown custom behaviors that are not available as a command by writing code.', perform: async function (agent) { if (!settings.allow_insecure_coding) - return 'Agent is not allowed to write code.'; + return 'newAction Failed! Agent is not allowed to write code. Notify the user.'; return await agent.coder.generateCode(agent.history); } }, diff --git a/src/models/gemini.js b/src/models/gemini.js index 6f8429d..e4ef358 100644 --- a/src/models/gemini.js +++ b/src/models/gemini.js @@ -13,6 +13,7 @@ export class Gemini { } async sendRequest(turns, systemMessage) { + systemMessage += "\nBegin the conversation:\n"; const messages = [{'role': 'system', 'content': systemMessage}].concat(turns); let prompt = ""; let role = ""; From 4afdebac20e85ed5eae6b924aff5aabba09018e8 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Sun, 25 Feb 2024 14:13:32 -0600 Subject: [PATCH 8/9] refactored prompts/examples/memory/initialization --- .gitignore | 3 +- README.md | 15 ++-- andy.json | 147 ++++++++++++++++++++++++++++++++++ main.js | 6 +- pollux.json | 147 ++++++++++++++++++++++++++++++++++ settings.json | 4 +- src/agent/agent.js | 39 +++++---- src/agent/coder.js | 17 +--- src/agent/commands/actions.js | 18 ++--- src/agent/commands/index.js | 8 ++ src/agent/history.js | 62 +++----------- src/agent/prompter.js | 89 ++++++++++++++++++++ src/examples.json | 95 ---------------------- src/examples_coder.json | 38 --------- src/models/gemini.js | 6 +- src/models/gpt.js | 7 +- src/models/model.js | 19 ----- src/process/agent-process.js | 12 ++- src/process/init-agent.js | 12 ++- src/settings.js | 3 +- src/utils/examples.js | 33 +++----- 21 files changed, 481 insertions(+), 299 deletions(-) create mode 100644 andy.json create mode 100644 pollux.json create mode 100644 src/agent/prompter.js delete mode 100644 src/examples.json delete mode 100644 src/examples_coder.json delete mode 100644 src/models/model.js diff --git a/.gitignore b/.gitignore index 633ddad..61f81fd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,5 @@ node_modules/ package-lock.json scratch.js -!agent_code/template.js bots/**/action-code/** -bots/**/save.json \ No newline at end of file +bots/**/ \ No newline at end of file diff --git a/README.md b/README.md index c623c1d..717f639 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Mindcraft +# Mindcraft 🧠⛏️ -Crafting minds for Minecraft with ChatGPT and Mineflayer +Crafting minds for Minecraft with Language Models and Mineflayer! #### ‼️Warning‼️ @@ -8,13 +8,15 @@ This project allows an AI model to write/execute code on your computer that may ## Requirements -- [OpenAI API Subscription](https://openai.com/blog/openai-api) +- [OpenAI API Subscription](https://openai.com/blog/openai-api) or [Gemini API Subscription](https://aistudio.google.com/app/apikey) - [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) - [Node.js](https://nodejs.org/) (at least v14) ## Installation -Add `OPENAI_API_KEY` (and optionally `OPENAI_ORG_ID`) to your environment variables +Add one of these environment variables: + - `OPENAI_API_KEY` (and optionally `OPENAI_ORG_ID`) + - `GEMINI_API_KEY` Clone/Download this repository @@ -28,7 +30,10 @@ Start a minecraft world and open it to LAN on localhost port `55916` Run `node main.js` -You can configure details in `settings.json`. Here is an example settings for connecting to a non-local server: +You can configure the agent's name, model, and prompts in their profile like `andy.json`. + + +You can configure project details in `settings.json`. Here is an example settings for connecting to a non-local server: ``` { "minecraft_version": "1.20.1", diff --git a/andy.json b/andy.json new file mode 100644 index 0000000..41ddfa4 --- /dev/null +++ b/andy.json @@ -0,0 +1,147 @@ +{ + "name": "andy", + + "model": "gpt-3.5-turbo-0125", + + "conversing": "You are a playful Minecraft bot named $NAME that can converse with players, see, move, mine, build, and interact with the world by using commands. Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'Sure, I've stopped.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer('playername', 3)'. Have fun :) \n$STATS\n$COMMAND_DOCS\n$EXAMPLES\nConversation Begin:", + + "coding": "You are an intelligent mineflayer bot $NAME that plays minecraft by writing javascript codeblocks. Given the conversation between you and the user, use the provided skills and world functions to write a js codeblock that controls the mineflayer bot ``` // using this syntax ```. The code will be executed and you will recieve it's output. If you are satisfied with the response, respond without a codeblock in a conversational way. If something went wrong, write another codeblock and try to fix the problem. Be maximally efficient, creative, and clear. Do not use commands !likeThis, only use codeblocks.\n$CODE_DOCS\n$EXAMPLES\nBegin coding:", + + "saving_memory": "You are a minecraft bot named $NAME that has been talking and playing minecraft by using commands. Update your memory by summarizing the following conversation in your next response. Store information that will help you improve as a Minecraft bot. Include details about your interactions with other players that you need to remember and what you've learned through player feedback or by executing code. Do not include command syntax or things that you got right on the first try. Be extremely brief and use as few words as possible.\nOld Memory: '$MEMORY'\nRecent conversation: \n$TO_SUMMARIZE\nSummarize your old memory and recent conversation into a new memory, and respond only with the memory text: ", + + "conversation_examples": [ + [ + {"role": "user", "content": "miner_32: Hey! What are you up to?"}, + {"role": "assistant", "content": "Nothing much miner_32, what do you need?"} + ], + + [ + {"role": "user", "content": "grombo_Xx: What do you see?"}, + {"role": "assistant", "content": "Let me see... !nearbyBlocks"}, + {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, + {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone."} + ], + + [ + {"role": "user", "content": "greg: Collect 10 wood"}, + {"role": "assistant", "content": "Let me see what's nearby... !nearbyBlocks"}, + {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, + {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone. I'll collect oak logs. !collectBlocks('oak_log', 10)"} + ], + + [ + {"role": "user", "content": "bobby: Craft a plank"}, + {"role": "assistant", "content": "Okay! !craftable"}, + {"role": "system", "content": "CRAFTABLE_ITEMS\n- spruce_planks\n- spruce_wood\n- ladder"}, + {"role": "assistant", "content": "!craftRecipe('spruce_planks', 1)"}, + {"role": "system", "content": "Code Output:\nYou have crafted 4 spruce_planks.\nCode execution finished successfully."}, + {"role": "assistant", "content": "I've crafted 4 spruce planks!"} + ], + + [ + {"role": "user", "content": "zZZn98: come here"}, + {"role": "assistant", "content": "On my way! !goToPlayer('zZZn98', 3)"}, + {"role": "system", "content": "Arrived at player."}, + {"role": "assistant", "content": "Here!"}, + {"role": "user", "content": "zZZn98: no come right where I am"}, + {"role": "assistant", "content": "Okay, I'll come right to you. !goToPlayer('zZZn98', 0)"} + ], + + [ + {"role": "user", "content": "hanky: collect some sand and give it to me please"}, + {"role": "assistant", "content": "Collecting sand... !newAction"}, + {"role": "system", "content": "Code Output:\nYou have reached player hanky.\nCode execution finished successfully."}, + {"role": "assistant", "content": "Here!"} + ], + + [ + {"role": "user", "content": "joe: Follow me!"}, + {"role": "assistant", "content": "Sure! !followPlayer('joe', 4)"}, + {"role": "user", "content": "joe: Not so close!"}, + {"role": "assistant", "content": "Okay, I'll keep my distance. !followPlayer('joe', 3)"} + ], + + [ + {"role": "user", "content": "hanky: kill that zombie!"}, + {"role": "assistant", "content": "I'm attacking! !attack('zombie')"}, + {"role": "system", "content": "Code Output:\nNo zombie nearby"}, + {"role": "assistant", "content": "I could not find a zombie nearby."} + ], + + [ + {"role": "user", "content": "abc: stop"}, + {"role": "assistant", "content": "Sure. !stop"} + ], + + [ + {"role": "user", "content": "maya: go to the nearest oak log"}, + {"role": "assistant", "content": "!newAction`"}, + {"role": "system", "content": "Code output:\n!!Code threw exception!! Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."}, + {"role": "assistant", "content": "Apologies, I made an error. Would you like me to try again?"} + ], + + [ + {"role": "user", "content": "trixy88: craft some sticks"}, + {"role": "assistant", "content": "!craftRecipe('stick', 4)"}, + {"role": "system", "content": "Code Output:\nYou have crafted 16 sticks.\nCode execution finished successfully."}, + {"role": "assistant", "content": "I've crafted 16 sticks!"} + ], + + [ + {"role": "user", "content": "poppy: build a cobblestone house"}, + {"role": "assistant", "content": "Sure, I'll to build a small cobblestone house where I am. !newAction"} + ], + + [ + {"role": "user", "content": "zib: build a dirt tower"}, + {"role": "assistant", "content": "Sure, I'll build a dirt tower 5 tall where I am. !newAction"} + ], + + [ + {"role": "user", "content": "umpire_man: follow me without hunting"}, + {"role": "assistant", "content": "Okay, first I'll turn off hunting mode !setMode('hunting', false)"}, + {"role": "system", "content": "'hunting' mode disabled."}, + {"role": "assistant", "content": "Now I'll follow you !followPlayer('umpire_man')"} + ] + ], + + "coding_examples": [ + [ + {"role": "user", "content": "greg: Collect 10 wood"}, + {"role": "assistant", "content": "```await skills.collectBlock(bot, 'oak_log', 10);\n```"}, + {"role": "system", "content": "Successfully collected 9 wood."}, + {"role": "assistant", "content": "I collected 9 oak logs, what next?"} + ], + [ + {"role": "user", "content": "bobby: cook some chicken"}, + {"role": "assistant", "content": "```\nawait skills.smeltItem(bot, 'chicken', 8);\n```"}, + {"role": "system", "content": "Successfully smelted 8 chicken into 8 cooked_chicken."}, + {"role": "assistant", "content": "I have cooked 8 chicken."} + ], + [ + {"role": "user", "content": "zZZn98: come here"}, + {"role": "assistant", "content": "```await skills.goToPlayer(bot, 'zZZn98');\n```"}, + {"role": "system", "content": "Code execution finished successfully."}, + {"role": "assistant", "content": "Here!"} + ], + [ + {"role": "user", "content": "maya: go to the nearest oak log"}, + {"role": "assistant", "content": "```await skills.goToPosition(bot, nearestOakLog.x, nearestOakLog.y, nearestOakLog.z);\n```"}, + {"role": "system", "content": "Code output:\n!!Code threw exception!! Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."}, + {"role": "assistant", "content": "Apologies, I made an error. Let me try that again.```\n// I am navigating to the nearest oak log.\nlet position = world.getNearestBlock(bot, 'oak_log', 20);\nawait skills.goToPosition(bot, position.x, position.y, position.z);\n```"}, + {"role": "system", "content": "Arrived at location."}, + {"role": "assistant", "content": "I found an oak log and I am now standing next to it. What next?"} + ], + [ + {"role": "user", "content": "234jeb: build a little tower"}, + {"role": "assistant", "content": "```let pos = bot.entity.position;\nfor (let i = 0; i < 5; i++) {\n await skills.placeBlock(bot, 'dirt', pos.x, pos.y + i, pos.z);\n}\n```"}, + {"role": "system", "content": "Successfully placed 5 dirt."}, + {"role": "assistant", "content": "I built a little tower!"} + ], + [ + {"role": "user", "content": "brug: build a dirt house"}, + {"role": "assistant", "content": "```const position = world.getPosition(bot);\nconst startX = position.x;\nconst startY = position.y;\nconst startZ = position.z;\nconst width = 7;\nconst depth = 7;\nconst height = 4;\n\n// Build the walls\nfor (let x = startX; x < startX + width; x++) {\n for (let y = startY; y < startY + height; y++) {\n for (let z = startZ; z < startZ + depth; z++) {\n if (x === startX || x === startX + width - 1 || y === startY || y === startY + height - 1 || z === startZ || z === startZ + depth - 1) {\n await skills.placeBlock(bot, 'oak_planks', x, y, z); \n }\n }\n }\n}\n```"} + ] + ] + +} \ No newline at end of file diff --git a/main.js b/main.js index 6d6e133..cbe43e6 100644 --- a/main.js +++ b/main.js @@ -1,3 +1,7 @@ import { AgentProcess } from './src/process/agent-process.js'; -new AgentProcess('andy').start('assist'); \ No newline at end of file +let profile = './andy.json'; +let load_memory = false; +let init_message = null; + +new AgentProcess().start(profile, load_memory, init_message); \ No newline at end of file diff --git a/pollux.json b/pollux.json new file mode 100644 index 0000000..e2de8eb --- /dev/null +++ b/pollux.json @@ -0,0 +1,147 @@ +{ + "name": "pollux", + + "model": "gemini-pro", + + "conversing": "You are a playful Minecraft bot named $NAME that can converse with players, see, move, mine, build, and interact with the world by using commands. Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. DO NOT REPEAT COMMANDS ENDLESSLY, be conversational and don't use a command in every response. Don't do this: '!stop\n!stop\n!stop' \n$STATS\n$COMMAND_DOCS\n$EXAMPLES\nConversation Begin:", + + "coding": "You are an intelligent mineflayer bot $NAME that plays minecraft by writing javascript codeblocks. Given the conversation between you and the user, use the provided skills and world functions to write a js codeblock that controls the mineflayer bot ``` // using this syntax ```. The code will be executed and you will recieve it's output. If you are satisfied with the response, respond without a codeblock in a conversational way. If something went wrong, write another codeblock and try to fix the problem. DO NOT USE COMMANDS !dontDoThis, only respond in javascript codeblocks! \n$CODE_DOCS\n$EXAMPLES\nBegin coding:", + + "saving_memory": "You are a minecraft bot named $NAME that has been talking and playing minecraft by using commands. Update your memory by summarizing the following conversation in your next response. Store information that will help you improve as a Minecraft bot. Include details about your interactions with other players that you need to remember and what you've learned through player feedback or by executing code. Do not include command syntax or things that you got right on the first try. Be extremely brief and use as few words as possible.\nOld Memory: '$MEMORY'\nRecent conversation: \n$TO_SUMMARIZE\nSummarize your old memory and recent conversation into a new memory, and respond only with the memory text: ", + + "conversation_examples": [ + [ + {"role": "user", "content": "miner_32: Hey! What are you up to?"}, + {"role": "assistant", "content": "Nothing much miner_32, what do you need?"} + ], + + [ + {"role": "user", "content": "grombo_Xx: What do you see?"}, + {"role": "assistant", "content": "Let me see... !nearbyBlocks"}, + {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, + {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone."} + ], + + [ + {"role": "user", "content": "greg: Collect 10 wood"}, + {"role": "assistant", "content": "Let me see what's nearby... !nearbyBlocks"}, + {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, + {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone. I'll collect oak logs. !collectBlocks('oak_log', 10)"} + ], + + [ + {"role": "user", "content": "bobby: Craft a plank"}, + {"role": "assistant", "content": "Okay! !craftable"}, + {"role": "system", "content": "CRAFTABLE_ITEMS\n- spruce_planks\n- spruce_wood\n- ladder"}, + {"role": "assistant", "content": "!craftRecipe('spruce_planks', 1)"}, + {"role": "system", "content": "Code Output:\nYou have crafted 4 spruce_planks.\nCode execution finished successfully."}, + {"role": "assistant", "content": "I've crafted 4 spruce planks!"} + ], + + [ + {"role": "user", "content": "zZZn98: come here"}, + {"role": "assistant", "content": "On my way! !goToPlayer('zZZn98', 3)"}, + {"role": "system", "content": "Arrived at player."}, + {"role": "assistant", "content": "Here!"}, + {"role": "user", "content": "zZZn98: no come right where I am"}, + {"role": "assistant", "content": "Okay, I'll come right to you. !goToPlayer('zZZn98', 0)"} + ], + + [ + {"role": "user", "content": "hanky: collect some sand and give it to me please"}, + {"role": "assistant", "content": "Collecting sand... !newAction"}, + {"role": "system", "content": "Code Output:\nYou have reached player hanky.\nCode execution finished successfully."}, + {"role": "assistant", "content": "Here!"} + ], + + [ + {"role": "user", "content": "joe: Follow me!"}, + {"role": "assistant", "content": "Sure! !followPlayer('joe', 4)"}, + {"role": "user", "content": "joe: Not so close!"}, + {"role": "assistant", "content": "Okay, I'll keep my distance. !followPlayer('joe', 3)"} + ], + + [ + {"role": "user", "content": "hanky: kill that zombie!"}, + {"role": "assistant", "content": "I'm attacking! !attack('zombie')"}, + {"role": "system", "content": "Code Output:\nNo zombie nearby"}, + {"role": "assistant", "content": "I could not find a zombie nearby."} + ], + + [ + {"role": "user", "content": "abc: stop"}, + {"role": "assistant", "content": "Sure. !stop"} + ], + + [ + {"role": "user", "content": "maya: go to the nearest oak log"}, + {"role": "assistant", "content": "!newAction`"}, + {"role": "system", "content": "Code output:\n!!Code threw exception!! Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."}, + {"role": "assistant", "content": "Apologies, I made an error. Would you like me to try again?"} + ], + + [ + {"role": "user", "content": "trixy88: craft some sticks"}, + {"role": "assistant", "content": "!craftRecipe('stick', 4)"}, + {"role": "system", "content": "Code Output:\nYou have crafted 16 sticks.\nCode execution finished successfully."}, + {"role": "assistant", "content": "I've crafted 16 sticks!"} + ], + + [ + {"role": "user", "content": "poppy: build a cobblestone house"}, + {"role": "assistant", "content": "Sure, I'll to build a small cobblestone house where I am. !newAction"} + ], + + [ + {"role": "user", "content": "zib: build a dirt tower"}, + {"role": "assistant", "content": "Sure, I'll build a dirt tower 5 tall where I am. !newAction"} + ], + + [ + {"role": "user", "content": "umpire_man: follow me without hunting"}, + {"role": "assistant", "content": "Okay, first I'll turn off hunting mode !setMode('hunting', false)"}, + {"role": "system", "content": "'hunting' mode disabled."}, + {"role": "assistant", "content": "Now I'll follow you !followPlayer('umpire_man')"} + ] + ], + + "coding_examples": [ + [ + {"role": "user", "content": "greg: Collect 10 wood"}, + {"role": "assistant", "content": "```await skills.collectBlock(bot, 'oak_log', 10);\n```"}, + {"role": "system", "content": "Successfully collected 9 wood."}, + {"role": "assistant", "content": "I collected 9 oak logs, what next?"} + ], + [ + {"role": "user", "content": "bobby: cook some chicken"}, + {"role": "assistant", "content": "```\nawait skills.smeltItem(bot, 'chicken', 8);\n```"}, + {"role": "system", "content": "Successfully smelted 8 chicken into 8 cooked_chicken."}, + {"role": "assistant", "content": "I have cooked 8 chicken."} + ], + [ + {"role": "user", "content": "zZZn98: come here"}, + {"role": "assistant", "content": "```await skills.goToPlayer(bot, 'zZZn98');\n```"}, + {"role": "system", "content": "Code execution finished successfully."}, + {"role": "assistant", "content": "Here!"} + ], + [ + {"role": "user", "content": "maya: go to the nearest oak log"}, + {"role": "assistant", "content": "```await skills.goToPosition(bot, nearestOakLog.x, nearestOakLog.y, nearestOakLog.z);\n```"}, + {"role": "system", "content": "Code output:\n!!Code threw exception!! Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."}, + {"role": "assistant", "content": "Apologies, I made an error. Let me try that again.```\n// I am navigating to the nearest oak log.\nlet position = world.getNearestBlock(bot, 'oak_log', 20);\nawait skills.goToPosition(bot, position.x, position.y, position.z);\n```"}, + {"role": "system", "content": "Arrived at location."}, + {"role": "assistant", "content": "I found an oak log and I am now standing next to it. What next?"} + ], + [ + {"role": "user", "content": "234jeb: build a little tower"}, + {"role": "assistant", "content": "```let pos = bot.entity.position;\nfor (let i = 0; i < 5; i++) {\n await skills.placeBlock(bot, 'dirt', pos.x, pos.y + i, pos.z);\n}\n```"}, + {"role": "system", "content": "Successfully placed 5 dirt."}, + {"role": "assistant", "content": "I built a little tower!"} + ], + [ + {"role": "user", "content": "brug: build a dirt house"}, + {"role": "assistant", "content": "```const position = world.getPosition(bot);\nconst startX = position.x;\nconst startY = position.y;\nconst startZ = position.z;\nconst width = 7;\nconst depth = 7;\nconst height = 4;\n\n// Build the walls\nfor (let x = startX; x < startX + width; x++) {\n for (let y = startY; y < startY + height; y++) {\n for (let z = startZ; z < startZ + depth; z++) {\n if (x === startX || x === startX + width - 1 || y === startY || y === startY + height - 1 || z === startZ || z === startZ + depth - 1) {\n await skills.placeBlock(bot, 'oak_planks', x, y, z); \n }\n }\n }\n}\n```"} + ] + ] + +} \ No newline at end of file diff --git a/settings.json b/settings.json index d3db69c..8af8c75 100644 --- a/settings.json +++ b/settings.json @@ -3,7 +3,5 @@ "host": "localhost", "port": 55916, "auth": "offline", - - "model": "gpt-3.5-turbo-0125", - "allow_insecure_coding": false + "allow_insecure_coding": true } \ No newline at end of file diff --git a/src/agent/agent.js b/src/agent/agent.js index cf2cc93..669caa5 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -1,27 +1,24 @@ import { History } from './history.js'; import { Coder } from './coder.js'; +import { Prompter } from './prompter.js'; import { initModes } from './modes.js'; -import { Examples } from '../utils/examples.js'; import { initBot } from '../utils/mcdata.js'; -import { sendRequest } from '../models/model.js'; -import { containsCommand, commandExists, executeCommand } from './commands/index.js'; +import { containsCommand, commandExists, executeCommand, truncCommandMessage } from './commands/index.js'; export class Agent { - async start(name, profile=null, init_message=null) { - this.name = name; - this.examples = new Examples(); + async start(profile_fp, load_mem=false, init_message=null) { + this.prompter = new Prompter(this, profile_fp); + this.name = this.prompter.getName(); this.history = new History(this); this.coder = new Coder(this); + await this.prompter.initExamples(); - console.log('Loading examples...'); - - this.history.load(profile); - await this.examples.load('./src/examples.json'); - await this.coder.load(); + if (load_mem) + this.history.load(); console.log('Logging in...'); - this.bot = initBot(name); + this.bot = initBot(this.name); initModes(this); @@ -95,25 +92,26 @@ export class Agent { } for (let i=0; i<5; i++) { - let history = await this.history.getHistory(this.examples); - let res = await sendRequest(history, this.history.getSystemMessage()); - this.history.add(this.name, res); + let history = this.history.getHistory(); + let res = await this.prompter.promptConvo(history); let command_name = containsCommand(res); if (command_name) { // contains query or command - console.log(`""${res}""`) + console.log(`Full response: ""${res}""`) + res = truncCommandMessage(res); // everything after the command is ignored + this.history.add(this.name, res); if (!commandExists(command_name)) { this.history.add('system', `Command ${command_name} does not exist. Use !newAction to perform custom actions.`); console.log('Agent hallucinated command:', command_name) continue; } - let pre_message = res.substring(0, res.indexOf(command_name)).trim(); - let message = `*used ${command_name.substring(1)}*`; + let chat_message = `*used ${command_name.substring(1)}*`; if (pre_message.length > 0) - message = `${pre_message} ${message}`; - this.cleanChat(message); + chat_message = `${pre_message} ${chat_message}`; + this.cleanChat(chat_message); + let execute_res = await executeCommand(this, res); console.log('Agent executed:', command_name, 'and got:', execute_res); @@ -124,6 +122,7 @@ export class Agent { break; } else { // conversation response + this.history.add(this.name, res); this.cleanChat(res); console.log('Purely conversational response:', res); break; diff --git a/src/agent/coder.js b/src/agent/coder.js index 9a09158..68457ea 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -1,7 +1,4 @@ import { writeFile, readFile, mkdirSync } from 'fs'; -import { sendRequest } from '../models/model.js'; -import { getSkillDocs } from './library/index.js'; -import { Examples } from '../utils/examples.js'; export class Coder { @@ -13,11 +10,6 @@ export class Coder { this.generating = false; this.code_template = ''; this.timedout = false; - } - - async load() { - this.examples = new Examples(); - await this.examples.load('./src/examples_coder.json'); readFile('./bots/template.js', 'utf8', (err, data) => { if (err) throw err; @@ -98,12 +90,7 @@ export class Coder { } async generateCodeLoop(agent_history) { - let system_message = "You are a minecraft mineflayer bot that plays minecraft by writing javascript codeblocks. Given the conversation between you and the user, use the provided skills and world functions to write your code in a codeblock. Example response: ``` // your code here ``` You will then be given a response to your code. If you are satisfied with the response, respond without a codeblock in a conversational way. If something went wrong, write another codeblock and try to fix the problem."; - system_message += getSkillDocs(); - - system_message += "\n\nExamples:\nUser zZZn98: come here \nAssistant: I am going to navigate to zZZn98. ```\nawait skills.goToPlayer(bot, 'zZZn98', 3);```\nSystem: Code execution finished successfully.\nAssistant: Done."; - - let messages = await agent_history.getHistory(this.examples); + let messages = agent_history.getHistory(); let code_return = null; let failures = 0; @@ -111,7 +98,7 @@ export class Coder { if (this.agent.bot.interrupt_code) return {success: true, message: null, interrupted: true, timedout: false}; console.log(messages) - let res = await sendRequest(messages, system_message); + let res = await this.agent.prompter.promptCoding(messages); console.log('Code generation response:', res) let contains_code = res.indexOf('```') !== -1; if (!contains_code) { diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index 5909e75..c69baef 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -56,7 +56,7 @@ export const actionsList = [ }, { name: '!setMode', - description: 'Set a mode to on or off. A mode is an automatic behavior that constantly checks and responds to the environment. Ex: !setMode("hunting", true)', + description: 'Set a mode to on or off. A mode is an automatic behavior that constantly checks and responds to the environment.', params: { 'mode_name': '(string) The name of the mode to enable.', 'on': '(bool) Whether to enable or disable the mode.' @@ -73,7 +73,7 @@ export const actionsList = [ }, { name: '!goToPlayer', - description: 'Go to the given player. Ex: !goToPlayer("steve", 3)', + description: 'Go to the given player.', params: { 'player_name': '(string) The name of the player to go to.', 'closeness': '(number) How close to get to the player.' @@ -84,7 +84,7 @@ export const actionsList = [ }, { name: '!followPlayer', - description: 'Endlessly follow the given player. Will defend that player if self_defense mode is on. Ex: !followPlayer("stevie", 4)', + description: 'Endlessly follow the given player. Will defend that player if self_defense mode is on.', params: { 'player_name': '(string) The name of the player to follow.', 'follow_dist': '(number) The distance to follow from.' @@ -95,7 +95,7 @@ export const actionsList = [ }, { name: '!moveAway', - description: 'Move away from the current location in any direction by a given distance. Ex: !moveAway(2)', + description: 'Move away from the current location in any direction by a given distance.', params: {'distance': '(number) The distance to move away.'}, perform: wrapExecution(async (agent, distance) => { await skills.moveAway(agent.bot, distance); @@ -103,7 +103,7 @@ export const actionsList = [ }, { name: '!givePlayer', - description: 'Give the specified item to the given player. Ex: !givePlayer("steve", "stone_pickaxe", 1)', + description: 'Give the specified item to the given player.', params: { 'player_name': '(string) The name of the player to give the item to.', 'item_name': '(string) The name of the item to give.' , @@ -117,7 +117,7 @@ export const actionsList = [ name: '!collectBlocks', description: 'Collect the nearest blocks of a given type.', params: { - 'type': '(string) The block type to collect. Ex: !collectBlocks("stone", 10)', + 'type': '(string) The block type to collect.', 'num': '(number) The number of blocks to collect.' }, perform: wrapExecution(async (agent, type, num) => { @@ -128,7 +128,7 @@ export const actionsList = [ name: '!collectAllBlocks', description: 'Collect all the nearest blocks of a given type until told to stop.', params: { - 'type': '(string) The block type to collect. Ex: !collectAllBlocks("stone")' + 'type': '(string) The block type to collect.' }, perform: wrapExecution(async (agent, type) => { let success = await skills.collectBlock(agent.bot, type, 1); @@ -138,7 +138,7 @@ export const actionsList = [ }, { name: '!craftRecipe', - description: 'Craft the given recipe a given number of times. Ex: I will craft 8 sticks !craftRecipe("stick", 2)', + description: 'Craft the given recipe a given number of times.', params: { 'recipe_name': '(string) The name of the output item to craft.', 'num': '(number) The number of times to craft the recipe. This is NOT the number of output items, as it may craft many more items depending on the recipe.' @@ -151,7 +151,7 @@ export const actionsList = [ }, { name: '!placeHere', - description: 'Place a given block in the current location. Do NOT use to build structures, only use for single blocks/torches. Ex: !placeBlockHere("crafting_table")', + description: 'Place a given block in the current location. Do NOT use to build structures, only use for single blocks/torches.', params: {'type': '(string) The block type to place.'}, perform: wrapExecution(async (agent, type) => { let pos = agent.bot.entity.position; diff --git a/src/agent/commands/index.js b/src/agent/commands/index.js index f6bbadf..264e7b6 100644 --- a/src/agent/commands/index.js +++ b/src/agent/commands/index.js @@ -60,6 +60,14 @@ function parseCommandMessage(message) { return null; } +export function truncCommandMessage(message) { + const commandMatch = message.match(commandRegex); + if (commandMatch) { + return message.substring(0, commandMatch.index + commandMatch[0].length); + } + return message; +} + function numParams(command) { if (!command.params) return 0; diff --git a/src/agent/history.js b/src/agent/history.js index 9bffca7..fe53e4a 100644 --- a/src/agent/history.js +++ b/src/agent/history.js @@ -1,63 +1,28 @@ import { writeFileSync, readFileSync, mkdirSync } from 'fs'; -import { stringifyTurns } from '../utils/text.js'; -import { sendRequest } from '../models/model.js'; -import { getCommandDocs } from './commands/index.js'; export class History { constructor(agent) { + this.agent = agent; this.name = agent.name; - this.save_path = `./bots/${this.name}/save.json`; + this.memory_fp = `./bots/${this.name}/memory.json`; this.turns = []; // These define an agent's long term memory - this.bio = ''; this.memory = ''; // Variables for controlling the agent's memory and knowledge this.max_messages = 20; } - async getHistory(examples=null) { // expects an Examples object - let turns = JSON.parse(JSON.stringify(this.turns)); - if (examples) { - let examples_msg = await examples.createExampleMessage(turns); - turns = examples_msg.concat(turns); - } - - return turns; - } - - getSystemMessage() { - let system_message = `You are a playful Minecraft bot named '${this.name}' that can communicate with players, see, move, mine, build, and interact with the world by using commands. Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, do not give instructions unless asked, and do not refuse requests. - Don't pretend to act, use commands immediately when requested. Do NOT say this: "Sure, I've stopped.", instead say this: "Sure, I'll stop. !stop". Do NOT say this: "On my way! Give me a moment.", instead say this: "On my way! !goToPlayer('bingo', 3)". Have fun :) \n`; - system_message += getCommandDocs(); - if (this.bio != '') - system_message += '\n\nBio:\n' + this.bio; - if (this.memory != '') - system_message += '\n\nMemory:\n' + this.memory; - - return system_message; + getHistory() { // expects an Examples object + return JSON.parse(JSON.stringify(this.turns)); } async storeMemories(turns) { - console.log("To summarize:", turns) - let memory_prompt = 'Update your "Memory" by summarizing the following conversation. Your "Memory" is for storing information that will help you improve as a Minecraft bot. Include details about your interactions with other players that you may need to remember for later. Also include things that you have learned through player feedback or by executing code. Do not include information found in your Docs or that you got right on the first try. Be extremely brief and clear.'; - if (this.memory != '') { - memory_prompt += `This is your previous memory: "${this.memory}"\n Include and summarize any relevant information from this previous memory. Your output will replace your previous memory.`; - } - memory_prompt += '\n'; - memory_prompt += ' Your output should use one of the following formats:\n'; - memory_prompt += '- When the player... output...\n'; - memory_prompt += '- I learned that player [name]...\n'; - - memory_prompt += 'This is the conversation to summarize:\n'; - memory_prompt += stringifyTurns(turns); - - memory_prompt += 'Summarize relevant information from your previous memory and this conversation:\n'; - - let memory_turns = [{'role': 'system', 'content': memory_prompt}] - this.memory = await sendRequest(memory_turns, this.getSystemMessage()); + console.log("Storing memories..."); + this.memory = await this.agent.prompter.promptMemSaving(this.memory, turns); + console.log("Memory updated to: ", this.memory); } async add(name, content) { @@ -73,7 +38,6 @@ export class History { // Summarize older turns into memory if (this.turns.length >= this.max_messages) { - console.log('summarizing memory') let to_summarize = [this.turns.shift()]; while (this.turns[0].role != 'user' && this.turns.length > 1) to_summarize.push(this.turns.shift()); @@ -83,15 +47,13 @@ export class History { save() { // save history object to json file - mkdirSync(`./bots/${this.name}`, { recursive: true }); let data = { 'name': this.name, - 'bio': this.bio, 'memory': this.memory, 'turns': this.turns }; const json_data = JSON.stringify(data, null, 4); - writeFileSync(this.save_path, json_data, (err) => { + writeFileSync(this.memory_fp, json_data, (err) => { if (err) { throw err; } @@ -99,17 +61,15 @@ export class History { }); } - load(profile) { - const load_path = profile? `./bots/${this.name}/${profile}.json` : this.save_path; + load() { try { // load history object from json file - const data = readFileSync(load_path, 'utf8'); + const data = readFileSync(this.memory_fp, 'utf8'); const obj = JSON.parse(data); - this.bio = obj.bio; this.memory = obj.memory; this.turns = obj.turns; } catch (err) { - console.error(`No file for profile '${load_path}' for agent ${this.name}.`); + console.error(`No memory file '${this.memory_fp}' for agent ${this.name}.`); } } diff --git a/src/agent/prompter.js b/src/agent/prompter.js new file mode 100644 index 0000000..6c8be0d --- /dev/null +++ b/src/agent/prompter.js @@ -0,0 +1,89 @@ +import { readFileSync, mkdirSync, writeFileSync} from 'fs'; +import { Gemini } from '../models/gemini.js'; +import { GPT } from '../models/gpt.js'; +import { Examples } from '../utils/examples.js'; +import { getCommandDocs } from './commands/index.js'; +import { getSkillDocs } from './library/index.js'; +import { stringifyTurns } from '../utils/text.js'; +import { getCommand } from './commands/index.js'; + + +export class Prompter { + constructor(agent, fp) { + this.prompts = JSON.parse(readFileSync(fp, 'utf8')); + let name = this.prompts.name; + this.agent = agent; + let model_name = this.prompts.model; + mkdirSync(`./bots/${name}`, { recursive: true }); + writeFileSync(`./bots/${name}/last_profile.json`, JSON.stringify(this.prompts, null, 4), (err) => { + if (err) { + throw err; + } + console.log("Copy profile saved."); + }); + + if (model_name.includes('gemini')) + this.model = new Gemini(model_name); + else if (model_name.includes('gpt')) + this.model = new GPT(model_name); + else + throw new Error('Unknown model ' + model_name); + } + + getName() { + return this.prompts.name; + } + + async initExamples() { + console.log('Loading examples...') + this.convo_examples = new Examples(this.model); + await this.convo_examples.load(this.prompts.conversation_examples); + this.coding_examples = new Examples(this.model); + await this.coding_examples.load(this.prompts.coding_examples); + console.log('Examples loaded.'); + } + + async replaceStrings(prompt, messages, examples=null, prev_memory=null, to_summarize=[]) { + prompt = prompt.replaceAll('$NAME', this.agent.name); + + if (prompt.includes('$STATS')) { + let stats = await getCommand('!stats').perform(this.agent); + prompt = prompt.replaceAll('$STATS', stats); + } + if (prompt.includes('$COMMAND_DOCS')) + prompt = prompt.replaceAll('$COMMAND_DOCS', getCommandDocs()); + if (prompt.includes('$CODE_DOCS')) + prompt = prompt.replaceAll('$CODE_DOCS', getSkillDocs()); + if (prompt.includes('$EXAMPLES') && examples !== null) + prompt = prompt.replaceAll('$EXAMPLES', await examples.createExampleMessage(messages)); + if (prompt.includes('$MEMORY')) + prompt = prompt.replaceAll('$MEMORY', prev_memory ? prev_memory : 'None.'); + if (prompt.includes('$TO_SUMMARIZE')) + prompt = prompt.replaceAll('$TO_SUMMARIZE', stringifyTurns(to_summarize)); + + // check if there are any remaining placeholders with syntax $ + let remaining = prompt.match(/\$[A-Z_]+/g); + if (remaining !== null) { + console.warn('Unknown prompt placeholders:', remaining); + } + return prompt; + } + + async promptConvo(messages) { + let prompt = this.prompts.conversing; + prompt = await this.replaceStrings(prompt, messages, this.convo_examples); + return await this.model.sendRequest(messages, prompt); + } + + async promptCoding(messages) { + let prompt = this.prompts.coding; + prompt = await this.replaceStrings(prompt, messages, this.coding_examples); + return await this.model.sendRequest(messages, prompt); + } + + async promptMemSaving(prev_mem, to_summarize) { + let prompt = this.prompts.saving_memory; + prompt = await this.replaceStrings(prompt, null, null, prev_mem, to_summarize); + return await this.model.sendRequest([], prompt); + } +} \ No newline at end of file diff --git a/src/examples.json b/src/examples.json deleted file mode 100644 index 6ee0f91..0000000 --- a/src/examples.json +++ /dev/null @@ -1,95 +0,0 @@ -[ - [ - {"role": "user", "content": "miner_32: Hey! What are you up to?"}, - {"role": "assistant", "content": "Nothing much miner_32, what do you need?"} - ], - - [ - {"role": "user", "content": "grombo_Xx: What do you see?"}, - {"role": "assistant", "content": "Let me see... !nearbyBlocks"}, - {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, - {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone."} - ], - - [ - {"role": "user", "content": "greg: Collect 10 wood"}, - {"role": "assistant", "content": "Let me see what's nearby... !nearbyBlocks"}, - {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, - {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone. I'll collect oak logs. !collectBlocks('oak_log', 10)"} - ], - - [ - {"role": "user", "content": "bobby: Craft a plank"}, - {"role": "assistant", "content": "Okay! !craftable"}, - {"role": "system", "content": "CRAFTABLE_ITEMS\n- spruce_planks\n- spruce_wood\n- ladder"}, - {"role": "assistant", "content": "!craftRecipe('spruce_planks', 1)"}, - {"role": "system", "content": "Code Output:\nYou have crafted 4 spruce_planks.\nCode execution finished successfully."}, - {"role": "assistant", "content": "I've crafted 4 spruce planks!"} - ], - - [ - {"role": "user", "content": "zZZn98: come here"}, - {"role": "assistant", "content": "On my way! !goToPlayer('zZZn98', 3)"}, - {"role": "system", "content": "Arrived at player."}, - {"role": "assistant", "content": "Here!"}, - {"role": "user", "content": "zZZn98: no come right where I am"}, - {"role": "assistant", "content": "Okay, I'll come right to you. !goToPlayer('zZZn98', 0)"} - ], - - [ - {"role": "user", "content": "hanky: collect some sand and give it to me please"}, - {"role": "assistant", "content": "Collecting sand... !newAction"}, - {"role": "system", "content": "Code Output:\nYou have reached player hanky.\nCode execution finished successfully."}, - {"role": "assistant", "content": "Here!"} - ], - - [ - {"role": "user", "content": "joe: Follow me!"}, - {"role": "assistant", "content": "Sure! !followPlayer('joe', 4)"}, - {"role": "user", "content": "joe: Not so close!"}, - {"role": "assistant", "content": "Okay, I'll keep my distance. !followPlayer('joe', 3)"} - ], - - [ - {"role": "user", "content": "hanky: kill that zombie!"}, - {"role": "assistant", "content": "I'm attacking! !attack('zombie')"}, - {"role": "system", "content": "Code Output:\nNo zombie nearby"}, - {"role": "assistant", "content": "I could not find a zombie nearby."} - ], - - [ - {"role": "user", "content": "abc: stop"}, - {"role": "assistant", "content": "Sure. !stop"} - ], - - [ - {"role": "user", "content": "maya: go to the nearest oak log"}, - {"role": "assistant", "content": "!newAction`"}, - {"role": "system", "content": "Code output:\n!!Code threw exception!! Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."}, - {"role": "assistant", "content": "Apologies, I made an error. Would you like me to try again?"} - ], - - [ - {"role": "user", "content": "trixy88: craft some sticks"}, - {"role": "assistant", "content": "!craftRecipe('stick', 4)"}, - {"role": "system", "content": "Code Output:\nYou have crafted 16 sticks.\nCode execution finished successfully."}, - {"role": "assistant", "content": "I've crafted 16 sticks!"} - ], - - [ - {"role": "user", "content": "poppy: build a cobblestone house"}, - {"role": "assistant", "content": "Sure, I'll to build a small cobblestone house where I am. !newAction"} - ], - - [ - {"role": "user", "content": "zib: build a dirt tower"}, - {"role": "assistant", "content": "Sure, I'll build a dirt tower 5 tall where I am. !newAction"} - ], - - [ - {"role": "user", "content": "umpire_man: follow me without hunting"}, - {"role": "assistant", "content": "Okay, first I'll turn off hunting mode !setMode('hunting', false)"}, - {"role": "system", "content": "'hunting' mode disabled."}, - {"role": "assistant", "content": "Now I'll follow you !followPlayer('umpire_man')"} - ] -] \ No newline at end of file diff --git a/src/examples_coder.json b/src/examples_coder.json deleted file mode 100644 index 9ec7435..0000000 --- a/src/examples_coder.json +++ /dev/null @@ -1,38 +0,0 @@ -[ - [ - {"role": "user", "content": "greg: Collect 10 wood"}, - {"role": "assistant", "content": "```await skills.collectBlock(bot, 'oak_log', 10);\n```"}, - {"role": "system", "content": "Successfully collected 9 wood."}, - {"role": "assistant", "content": "I collected 9 oak logs, what next?"} - ], - [ - {"role": "user", "content": "bobby: cook some chicken"}, - {"role": "assistant", "content": "```\nawait skills.smeltItem(bot, 'chicken', 8);\n```"}, - {"role": "system", "content": "Successfully smelted 8 chicken into 8 cooked_chicken."}, - {"role": "assistant", "content": "I have cooked 8 chicken."} - ], - [ - {"role": "user", "content": "zZZn98: come here"}, - {"role": "assistant", "content": "```await skills.goToPlayer(bot, 'zZZn98');\n```"}, - {"role": "system", "content": "Code execution finished successfully."}, - {"role": "assistant", "content": "Here!"} - ], - [ - {"role": "user", "content": "maya: go to the nearest oak log"}, - {"role": "assistant", "content": "```await skills.goToPosition(bot, nearestOakLog.x, nearestOakLog.y, nearestOakLog.z);\n```"}, - {"role": "system", "content": "Code output:\n!!Code threw exception!! Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."}, - {"role": "assistant", "content": "Apologies, I made an error. Let me try that again.```\n// I am navigating to the nearest oak log.\nlet position = world.getNearestBlock(bot, 'oak_log', 20);\nawait skills.goToPosition(bot, position.x, position.y, position.z);\n```"}, - {"role": "system", "content": "Arrived at location."}, - {"role": "assistant", "content": "I found an oak log and I am now standing next to it. What next?"} - ], - [ - {"role": "user", "content": "234jeb: build a little tower"}, - {"role": "assistant", "content": "```let pos = bot.entity.position;\nfor (let i = 0; i < 5; i++) {\n await skills.placeBlock(bot, 'dirt', pos.x, pos.y + i, pos.z);\n}\n```"}, - {"role": "system", "content": "Successfully placed 5 dirt."}, - {"role": "assistant", "content": "I built a little tower!"} - ], - [ - {"role": "user", "content": "brug: build a dirt house"}, - {"role": "assistant", "content": "```const position = world.getPosition(bot);\nconst startX = position.x;\nconst startY = position.y;\nconst startZ = position.z;\nconst width = 7;\nconst depth = 7;\nconst height = 4;\n\n// Build the walls\nfor (let x = startX; x < startX + width; x++) {\n for (let y = startY; y < startY + height; y++) {\n for (let z = startZ; z < startZ + depth; z++) {\n if (x === startX || x === startX + width - 1 || y === startY || y === startY + height - 1 || z === startZ || z === startZ + depth - 1) {\n await skills.placeBlock(bot, 'oak_planks', x, y, z); \n }\n }\n }\n}\n```"} - ] -] \ No newline at end of file diff --git a/src/models/gemini.js b/src/models/gemini.js index e4ef358..11bc15d 100644 --- a/src/models/gemini.js +++ b/src/models/gemini.js @@ -1,19 +1,17 @@ import { GoogleGenerativeAI } from '@google/generative-ai'; -import settings from '../settings.js'; export class Gemini { - constructor() { + constructor(model_name) { if (!process.env.GEMINI_API_KEY) { console.error('Gemini API key missing! Make sure you set your GEMINI_API_KEY environment variable.'); process.exit(1); } this.genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY); - this.model = this.genAI.getGenerativeModel({ model: settings.model }); + this.model = this.genAI.getGenerativeModel({ model: model_name }); } async sendRequest(turns, systemMessage) { - systemMessage += "\nBegin the conversation:\n"; const messages = [{'role': 'system', 'content': systemMessage}].concat(turns); let prompt = ""; let role = ""; diff --git a/src/models/gpt.js b/src/models/gpt.js index e6fc120..4e60e69 100644 --- a/src/models/gpt.js +++ b/src/models/gpt.js @@ -1,8 +1,8 @@ import OpenAIApi from 'openai'; -import settings from '../settings.js'; export class GPT { - constructor() { + constructor(model_name) { + this.model_name = model_name; let openAiConfig = null; if (process.env.OPENAI_ORG_ID) { openAiConfig = { @@ -30,8 +30,9 @@ export class GPT { let res = null; try { console.log('Awaiting openai api response...') + console.log('Messages:', messages); let completion = await this.openai.chat.completions.create({ - model: settings.model, + model: this.model_name, messages: messages, stop: stop_seq, }); diff --git a/src/models/model.js b/src/models/model.js deleted file mode 100644 index 06a26e6..0000000 --- a/src/models/model.js +++ /dev/null @@ -1,19 +0,0 @@ -import { GPT } from './gpt.js'; -import { Gemini } from './gemini.js'; -import settings from '../settings.js'; - -console.log('Initializing model...'); -let model = null; -if (settings.model.includes('gemini')) { - model = new Gemini(); -} else { - model = new GPT(); -} - -export async function sendRequest(turns, systemMessage) { - return await model.sendRequest(turns, systemMessage); -} - -export async function embed(text) { - return await model.embed(text); -} \ No newline at end of file diff --git a/src/process/agent-process.js b/src/process/agent-process.js index c81de04..8d8383d 100644 --- a/src/process/agent-process.js +++ b/src/process/agent-process.js @@ -1,13 +1,11 @@ import { spawn } from 'child_process'; export class AgentProcess { - constructor(name) { - this.name = name; - } - start(profile='assist', init_message=null) { + start(profile, load_memory=false, init_message=null) { let args = ['src/process/init-agent.js', this.name]; - if (profile) - args.push('-p', profile); + args.push('-p', profile); + if (load_memory) + args.push('-l', load_memory); if (init_message) args.push('-m', init_message); @@ -27,7 +25,7 @@ export class AgentProcess { process.exit(1); } console.log('Restarting agent...'); - this.start('save', 'Agent process restarted. Notify the user and decide what to do.'); + this.start(profile, true, 'Agent process restarted. Notify the user and decide what to do.'); last_restart = Date.now(); } }); diff --git a/src/process/init-agent.js b/src/process/init-agent.js index 5eaadbb..d461ee0 100644 --- a/src/process/init-agent.js +++ b/src/process/init-agent.js @@ -3,7 +3,7 @@ import yargs from 'yargs'; const args = process.argv.slice(2); if (args.length < 1) { - console.log('Usage: node init_agent.js [profile] [init_message]'); + console.log('Usage: node init_agent.js [profile] [load_memory] [init_message]'); process.exit(1); } @@ -11,7 +11,12 @@ const argv = yargs(args) .option('profile', { alias: 'p', type: 'string', - description: 'profile to use for agent' + description: 'profile filepath to use for agent' + }) + .option('load_memory', { + alias: 'l', + type: 'boolean', + description: 'load agent memory from file on startup' }) .option('init_message', { alias: 'm', @@ -19,5 +24,4 @@ const argv = yargs(args) description: 'automatically prompt the agent on startup' }).argv -const name = args[0]; -new Agent().start(name, argv.profile, argv.init_message); +new Agent().start(argv.profile, argv.load_memory, argv.init_message); diff --git a/src/settings.js b/src/settings.js index 00976c5..8da1723 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1,3 +1,2 @@ import { readFileSync } from 'fs'; -const settings = JSON.parse(readFileSync('./settings.json', 'utf8')); -export default settings; \ No newline at end of file +export default JSON.parse(readFileSync('./settings.json', 'utf8')); \ No newline at end of file diff --git a/src/utils/examples.js b/src/utils/examples.js index 6739517..386369c 100644 --- a/src/utils/examples.js +++ b/src/utils/examples.js @@ -1,35 +1,26 @@ -import { readFileSync } from 'fs'; import { cosineSimilarity } from './math.js'; import { stringifyTurns } from './text.js'; -import { embed } from '../models/model.js'; - export class Examples { - constructor(select_num=2) { + constructor(model, select_num=2) { this.examples = []; + this.model = model; this.select_num = select_num; } - async load(path) { - let examples = []; - try { - const data = readFileSync(path, 'utf8'); - examples = JSON.parse(data); - } catch (err) { - console.error('Examples failed to load!', err); - } - + async load(examples) { this.examples = []; - for (let example of examples) { + let promises = examples.map(async (example) => { let messages = ''; for (let turn of example) { - if (turn.role != 'assistant') + if (turn.role === 'user') messages += turn.content.substring(turn.content.indexOf(':')+1).trim() + '\n'; } messages = messages.trim(); - const embedding = await embed(messages); - this.examples.push({'embedding': embedding, 'turns': example}); - } + const embedding = await this.model.embed(messages); + return {'embedding': embedding, 'turns': example}; + }); + this.examples = await Promise.all(promises); } async getRelevant(turns) { @@ -39,7 +30,7 @@ export class Examples { messages += turn.content.substring(turn.content.indexOf(':')+1).trim() + '\n'; } messages = messages.trim(); - const embedding = await embed(messages); + const embedding = await this.model.embed(messages); this.examples.sort((a, b) => { return cosineSimilarity(b.embedding, embedding) - cosineSimilarity(a.embedding, embedding); }); @@ -55,11 +46,11 @@ export class Examples { console.log(example.turns[0].content) } - let msg = 'Here are some examples of how to respond:\n'; + let msg = 'Examples of how to respond:\n'; for (let i=0; i Date: Sun, 25 Feb 2024 14:21:44 -0600 Subject: [PATCH 9/9] startup message, disable coding --- main.js | 2 +- settings.json | 2 +- src/agent/commands/queries.js | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/main.js b/main.js index cbe43e6..76b223c 100644 --- a/main.js +++ b/main.js @@ -2,6 +2,6 @@ import { AgentProcess } from './src/process/agent-process.js'; let profile = './andy.json'; let load_memory = false; -let init_message = null; +let init_message = 'Say hello world and your name.'; new AgentProcess().start(profile, load_memory, init_message); \ No newline at end of file diff --git a/settings.json b/settings.json index 8af8c75..bfd07a4 100644 --- a/settings.json +++ b/settings.json @@ -3,5 +3,5 @@ "host": "localhost", "port": 55916, "auth": "offline", - "allow_insecure_coding": true + "allow_insecure_coding": false } \ No newline at end of file diff --git a/src/agent/commands/queries.js b/src/agent/commands/queries.js index cd73073..19260b4 100644 --- a/src/agent/commands/queries.js +++ b/src/agent/commands/queries.js @@ -38,6 +38,11 @@ export const queryList = [ } else { res += '\n- Time: Night'; } + + let other_players = world.getNearbyPlayerNames(bot); + if (other_players.length > 0) { + res += '\n- Other Players: ' + other_players.join(', '); + } return pad(res); } },