From 09b2c23cc1d1d98e6ff422c1e84d1dcf2006c338 Mon Sep 17 00:00:00 2001 From: bartek Date: Wed, 23 Oct 2024 11:03:50 +0200 Subject: [PATCH 01/59] Update README.md run in docker instruction --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index d2ef957..7aa15be 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,12 @@ You can configure the agent's name, model, and prompts in their profile like `an You can configure project details in `settings.js`. [See file for more details](settings.js) +### Run in docker to reduce some of the risks + +``` +docker run -i -t --rm -v $(pwd):/usr/src/app -w /usr/src/app node:latest npm start +``` + ### Online Servers To connect to online servers your bot will need an official Microsoft/Minecraft account. You can use your own personal one, but will need another account if you want to connect with it. Here are example settings for this: ``` From e6afe2cfeca5cdcf02a1193c510378e3604556d3 Mon Sep 17 00:00:00 2001 From: Kevin Ji <1146876+kevinji@users.noreply.github.com> Date: Thu, 24 Oct 2024 12:53:16 -0700 Subject: [PATCH 02/59] actions: Fix typo in description --- src/agent/commands/actions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index 246d649..fc562a8 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -265,7 +265,7 @@ export const actionsList = [ }, { name: '!clearFurnace', - description: 'Tak all items out of the nearest furnace.', + description: 'Take all items out of the nearest furnace.', params: { }, perform: wrapExecution(async (agent) => { await skills.clearNearestFurnace(agent.bot); From 1b690c00cf4c47f0f7a8539aa32e963dd582c250 Mon Sep 17 00:00:00 2001 From: bartek Date: Fri, 25 Oct 2024 10:30:32 +0200 Subject: [PATCH 03/59] Update README.md more instruction --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 7aa15be..c0e46da 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,18 @@ You can configure project details in `settings.js`. [See file for more details]( ### Run in docker to reduce some of the risks +If you intent to `allow_insecure_coding` it might be a good idea to put the whole app into a docker container to reduce risks of damaging your system via a malicious code. + ``` docker run -i -t --rm -v $(pwd):/usr/src/app -w /usr/src/app node:latest npm start ``` +When running in docker, if you want the bot to join your local minecraft server, you have to use special host address to call your localhost from your docker container: `host.docker.internal`. Put this into your `settings.json`: + +``` +"host": "host.docker.internal", // instead of "localhost", to join your local miecraft from inside the docker container +``` + ### Online Servers To connect to online servers your bot will need an official Microsoft/Minecraft account. You can use your own personal one, but will need another account if you want to connect with it. Here are example settings for this: ``` From c0fbcb27aa095e83840102fe54be33c9806b502d Mon Sep 17 00:00:00 2001 From: bartek Date: Fri, 25 Oct 2024 10:45:49 +0200 Subject: [PATCH 04/59] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c0e46da..c9b102c 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ If you intent to `allow_insecure_coding` it might be a good idea to put the whol docker run -i -t --rm -v $(pwd):/usr/src/app -w /usr/src/app node:latest npm start ``` -When running in docker, if you want the bot to join your local minecraft server, you have to use special host address to call your localhost from your docker container: `host.docker.internal`. Put this into your `settings.json`: +When running in docker, if you want the bot to join your local minecraft server, you have to use a special host address to call your localhost from inside your docker container: `host.docker.internal`. Put this into your [settings.js](settings.js): ``` "host": "host.docker.internal", // instead of "localhost", to join your local miecraft from inside the docker container From 44350d3c243825f5e90b502feb94e01899c5cd5c Mon Sep 17 00:00:00 2001 From: bartek Date: Fri, 25 Oct 2024 10:52:58 +0200 Subject: [PATCH 05/59] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c9b102c..1e9f5cd 100644 --- a/README.md +++ b/README.md @@ -48,16 +48,16 @@ You can configure project details in `settings.js`. [See file for more details]( ### Run in docker to reduce some of the risks -If you intent to `allow_insecure_coding` it might be a good idea to put the whole app into a docker container to reduce risks of damaging your system via a malicious code. +If you intent to `allow_insecure_coding`, it might be a good idea to put the whole app into a docker container to reduce risks of running unknown code. ``` -docker run -i -t --rm -v $(pwd):/usr/src/app -w /usr/src/app node:latest npm start +docker run -i -t --rm -v $(pwd):/app -w /app node:latest node main.js ``` -When running in docker, if you want the bot to join your local minecraft server, you have to use a special host address to call your localhost from inside your docker container: `host.docker.internal`. Put this into your [settings.js](settings.js): +When running in docker, if you want the bot to join your local minecraft server, you have to use a special host address `host.docker.internal` to call your localhost from inside your docker container. Put this into your [settings.js](settings.js): ``` -"host": "host.docker.internal", // instead of "localhost", to join your local miecraft from inside the docker container +"host": "host.docker.internal", // instead of "localhost", to join your local minecraft from inside the docker container ``` ### Online Servers From b11416d0fcbe05801b40e47f6f19c880d6e4432f Mon Sep 17 00:00:00 2001 From: kumavis Date: Fri, 25 Oct 2024 15:28:27 -1000 Subject: [PATCH 06/59] refactor: add TaskManager, move task mgmt out of coder --- src/agent/agent.js | 23 ++++-- src/agent/coder.js | 130 +++-------------------------- src/agent/commands/actions.js | 98 +++++++++++----------- src/agent/commands/index.js | 5 -- src/agent/modes.js | 6 +- src/agent/npc/build_goal.js | 2 +- src/agent/npc/controller.js | 10 +-- src/agent/npc/item_goal.js | 4 +- src/agent/self_prompter.js | 2 +- src/agent/tasks.js | 148 ++++++++++++++++++++++++++++++++++ 10 files changed, 233 insertions(+), 195 deletions(-) create mode 100644 src/agent/tasks.js diff --git a/src/agent/agent.js b/src/agent/agent.js index 5be2385..30715d8 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -4,6 +4,7 @@ import { Prompter } from './prompter.js'; import { initModes } from './modes.js'; import { initBot } from '../utils/mcdata.js'; import { containsCommand, commandExists, executeCommand, truncCommandMessage, isAction } from './commands/index.js'; +import { TaskManager } from './tasks.js'; import { NPCContoller } from './npc/controller.js'; import { MemoryBank } from './memory_bank.js'; import { SelfPrompter } from './self_prompter.js'; @@ -13,6 +14,7 @@ import settings from '../../settings.js'; export class Agent { async start(profile_fp, load_mem=false, init_message=null, count_id=0) { + this.tasks = new TaskManager(this); this.prompter = new Prompter(this, profile_fp); this.name = this.prompter.getName(); this.history = new History(this); @@ -40,7 +42,7 @@ export class Agent { await new Promise((resolve) => setTimeout(resolve, 1000)); console.log(`${this.name} spawned.`); - this.coder.clear(); + this.clearBotLogs(); const ignore_messages = [ "Set own game mode to", @@ -91,6 +93,17 @@ export class Agent { }); } + interruptBot() { + this.bot.interrupt_code = true; + this.bot.collectBlock.cancelTask(); + this.bot.pathfinder.stop(); + this.bot.pvp.stop(); + } + + clearBotLogs() { + this.bot.output = ''; + this.bot.interrupt_code = false; + } async cleanChat(message, translate_up_to=-1) { let to_translate = message; @@ -250,8 +263,8 @@ export class Agent { this.cleanKill('Bot disconnected! Killing agent process.'); }); this.bot.on('death', () => { - this.coder.cancelResume(); - this.coder.stop(); + this.tasks.cancelResume(); + this.tasks.stop(); }); this.bot.on('kicked', (reason) => { console.warn('Bot kicked!', reason); @@ -267,7 +280,7 @@ export class Agent { this.bot.clearControlStates(); this.bot.pathfinder.stop(); // clear any lingering pathfinder this.bot.modes.unPauseAll(); - this.coder.executeResume(); + this.tasks.resumeTask(); }); // Init NPC controller @@ -297,7 +310,7 @@ export class Agent { } isIdle() { - return !this.coder.executing && !this.coder.generating; + return !this.tasks.executing && !this.coder.generating; } cleanKill(msg='Killing agent process...') { diff --git a/src/agent/coder.js b/src/agent/coder.js index d312387..e1179d3 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -7,11 +7,8 @@ export class Coder { this.agent = agent; this.file_counter = 0; this.fp = '/bots/'+agent.name+'/action-code/'; - this.executing = false; this.generating = false; this.code_template = ''; - this.timedout = false; - this.cur_action_name = ''; readFile('./bots/template.js', 'utf8', (err, data) => { if (err) throw err; @@ -83,7 +80,7 @@ export class Coder { async generateCode(agent_history) { // wrapper to prevent overlapping code generation loops - await this.stop(); + await this.agent.tasks.stop(); this.generating = true; let res = await this.generateCodeLoop(agent_history); this.generating = false; @@ -119,7 +116,7 @@ export class Coder { } if (failures >= 3) { - return {success: false, message: 'Action failed, agent would not write code.', interrupted: false, timedout: false}; + return { success: false, message: 'Action failed, agent would not write code.', interrupted: false, timedout: false }; } messages.push({ role: 'system', @@ -137,22 +134,22 @@ export class Coder { continue; } - const execution_file = await this.stageCode(code); - if (!execution_file) { + const executionModuleExports = await this.stageCode(code); + if (!executionModuleExports) { agent_history.add('system', 'Failed to stage code, something is wrong.'); return {success: false, message: null, interrupted: false, timedout: false}; } - code_return = await this.execute(async ()=>{ - return await execution_file.main(this.agent.bot); - }, settings.code_timeout_mins); + code_return = await this.agent.tasks.runTask('newAction', async () => { + return await executionModuleExports.main(this.agent.bot); + }, { timeout: settings.code_timeout_mins }); if (code_return.interrupted && !code_return.timedout) - return {success: false, message: null, interrupted: true, timedout: false}; + return { success: false, message: null, interrupted: true, timedout: false }; console.log("Code generation result:", code_return.success, code_return.message); if (code_return.success) { const summary = "Summary of newAction\nAgent wrote this code: \n```" + this.sanitizeCode(code) + "```\nCode Output:\n" + code_return.message; - return {success: true, message: summary, interrupted: false, timedout: false}; + return { success: true, message: summary, interrupted: false, timedout: false }; } messages.push({ @@ -164,114 +161,7 @@ export class Coder { content: code_return.message + '\nCode failed. Please try again:' }); } - return {success: false, message: null, interrupted: false, timedout: true}; + return { success: false, message: null, interrupted: false, timedout: true }; } - async executeResume(func=null, timeout=10) { - const new_resume = func != null; - if (new_resume) { // start new resume - this.resume_func = func; - this.resume_name = this.cur_action_name; - } - if (this.resume_func != null && this.agent.isIdle() && (!this.agent.self_prompter.on || new_resume)) { - this.cur_action_name = this.resume_name; - let res = await this.execute(this.resume_func, timeout); - this.cur_action_name = ''; - return res; - } else { - return {success: false, message: null, interrupted: false, timedout: false}; - } - } - - cancelResume() { - this.resume_func = null; - this.resume_name = null; - } - - setCurActionName(name) { - this.cur_action_name = name.replace(/!/g, ''); - } - - // returns {success: bool, message: string, interrupted: bool, timedout: false} - async execute(func, timeout=10) { - if (!this.code_template) return {success: false, message: "Code template not loaded.", interrupted: false, timedout: false}; - - let TIMEOUT; - try { - console.log('executing code...\n'); - await this.stop(); - this.clear(); - - this.executing = true; - if (timeout > 0) - TIMEOUT = this._startTimeout(timeout); - await func(); // open fire - this.executing = false; - clearTimeout(TIMEOUT); - - let output = this.formatOutput(this.agent.bot); - let interrupted = this.agent.bot.interrupt_code; - let timedout = this.timedout; - this.clear(); - if (!interrupted && !this.generating) this.agent.bot.emit('idle'); - return {success:true, message: output, interrupted, timedout}; - } catch (err) { - this.executing = false; - clearTimeout(TIMEOUT); - this.cancelResume(); - console.error("Code execution triggered catch: " + err); - await this.stop(); - - let message = this.formatOutput(this.agent.bot) + '!!Code threw exception!! Error: ' + err; - let interrupted = this.agent.bot.interrupt_code; - this.clear(); - if (!interrupted && !this.generating) this.agent.bot.emit('idle'); - return {success: false, message, interrupted, timedout: false}; - } - } - - formatOutput(bot) { - if (bot.interrupt_code && !this.timedout) return ''; - let output = bot.output; - const MAX_OUT = 500; - if (output.length > MAX_OUT) { - output = `Code output is very long (${output.length} chars) and has been shortened.\n - First outputs:\n${output.substring(0, MAX_OUT/2)}\n...skipping many lines.\nFinal outputs:\n ${output.substring(output.length - MAX_OUT/2)}`; - } - else { - output = 'Code output:\n' + output; - } - return output; - } - - async stop() { - if (!this.executing) return; - const start = Date.now(); - while (this.executing) { - this.agent.bot.interrupt_code = true; - this.agent.bot.collectBlock.cancelTask(); - this.agent.bot.pathfinder.stop(); - this.agent.bot.pvp.stop(); - console.log('waiting for code to finish executing...'); - await new Promise(resolve => setTimeout(resolve, 1000)); - if (Date.now() - start > 10 * 1000) { - this.agent.cleanKill('Code execution refused stop after 10 seconds. Killing process.'); - } - } - } - - clear() { - this.agent.bot.output = ''; - this.agent.bot.interrupt_code = false; - this.timedout = false; - } - - _startTimeout(TIMEOUT_MINS=10) { - return setTimeout(async () => { - console.warn(`Code execution timed out after ${TIMEOUT_MINS} minutes. Attempting force stop.`); - this.timedout = true; - this.agent.history.add('system', `Code execution timed out after ${TIMEOUT_MINS} minutes. Attempting force stop.`); - await this.stop(); // last attempt to stop - }, TIMEOUT_MINS*60*1000); - } } \ No newline at end of file diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index a80ad29..d8e0cfe 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -1,17 +1,12 @@ import * as skills from '../library/skills.js'; import settings from '../../../settings.js'; -function wrapExecution(func, resume=false, timeout=-1) { +function runAsTask (taskLabel, taskFn, resume = false, timeout = -1) { return async function (agent, ...args) { - let code_return; - const wrappedFunction = async () => { - await func(agent, ...args); + const taskFnWithAgent = async () => { + await taskFn(agent, ...args); }; - if (resume) { - code_return = await agent.coder.executeResume(wrappedFunction, timeout); - } else { - code_return = await agent.coder.execute(wrappedFunction, timeout); - } + const code_return = await agent.tasks.runTask(`action:${taskLabel}`, taskFnWithAgent, { timeout, resume }); if (code_return.interrupted && !code_return.timedout) return; return code_return.message; @@ -36,9 +31,9 @@ export const actionsList = [ name: '!stop', description: 'Force stop all actions and commands that are currently executing.', perform: async function (agent) { - await agent.coder.stop(); - agent.coder.clear(); - agent.coder.cancelResume(); + await agent.tasks.stop(); + agent.clearBotLogs(); + agent.tasks.cancelResume(); agent.bot.emit('idle'); let msg = 'Agent stopped.'; if (agent.self_prompter.on) @@ -78,7 +73,7 @@ export const actionsList = [ 'player_name': {type: 'string', description: 'The name of the player to go to.'}, 'closeness': {type: 'float', description: 'How close to get to the player.', domain: [0, Infinity]} }, - perform: wrapExecution(async (agent, player_name, closeness) => { + perform: runAsTask('goToPlayer', async (agent, player_name, closeness) => { return await skills.goToPlayer(agent.bot, player_name, closeness); }) }, @@ -89,7 +84,7 @@ export const actionsList = [ 'player_name': {type: 'string', description: 'name of the player to follow.'}, 'follow_dist': {type: 'float', description: 'The distance to follow from.', domain: [0, Infinity]} }, - perform: wrapExecution(async (agent, player_name, follow_dist) => { + perform: runAsTask('followPlayer', async (agent, player_name, follow_dist) => { await skills.followPlayer(agent.bot, player_name, follow_dist); }, true) }, @@ -101,7 +96,7 @@ export const actionsList = [ 'closeness': { type: 'float', description: 'How close to get to the block.', domain: [0, Infinity] }, 'search_range': { type: 'float', description: 'The distance to search for the block.', domain: [0, Infinity] } }, - perform: wrapExecution(async (agent, type, closeness, range) => { + perform: runAsTask('goToBlock', async (agent, type, closeness, range) => { await skills.goToNearestBlock(agent.bot, type, closeness, range); }) }, @@ -109,7 +104,7 @@ export const actionsList = [ name: '!moveAway', description: 'Move away from the current location in any direction by a given distance.', params: {'distance': { type: 'float', description: 'The distance to move away.', domain: [0, Infinity] }}, - perform: wrapExecution(async (agent, distance) => { + perform: runAsTask('moveAway', async (agent, distance) => { await skills.moveAway(agent.bot, distance); }) }, @@ -127,11 +122,11 @@ export const actionsList = [ name: '!goToPlace', description: 'Go to a saved location.', params: {'name': { type: 'string', description: 'The name of the location to go to.' }}, - perform: wrapExecution(async (agent, name) => { + perform: runAsTask('goToPlace', async (agent, name) => { const pos = agent.memory_bank.recallPlace(name); if (!pos) { - skills.log(agent.bot, `No location named "${name}" saved.`); - return; + skills.log(agent.bot, `No location named "${name}" saved.`); + return; } await skills.goToPosition(agent.bot, pos[0], pos[1], pos[2], 1); }) @@ -144,7 +139,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the item to give.' }, 'num': { type: 'int', description: 'The number of items to give.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: wrapExecution(async (agent, player_name, item_name, num) => { + perform: runAsTask('givePlayer', async (agent, player_name, item_name, num) => { await skills.giveToPlayer(agent.bot, item_name, player_name, num); }) }, @@ -152,7 +147,7 @@ export const actionsList = [ name: '!consume', description: 'Eat/drink the given item.', params: {'item_name': { type: 'ItemName', description: 'The name of the item to consume.' }}, - perform: wrapExecution(async (agent, item_name) => { + perform: runAsTask('consume', async (agent, item_name) => { await agent.bot.consume(item_name); skills.log(agent.bot, `Consumed ${item_name}.`); }) @@ -161,7 +156,7 @@ export const actionsList = [ name: '!equip', description: 'Equip the given item.', params: {'item_name': { type: 'ItemName', description: 'The name of the item to equip.' }}, - perform: wrapExecution(async (agent, item_name) => { + perform: runAsTask('equip', async (agent, item_name) => { await skills.equip(agent.bot, item_name); }) }, @@ -172,7 +167,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the item to put in the chest.' }, 'num': { type: 'int', description: 'The number of items to put in the chest.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: wrapExecution(async (agent, item_name, num) => { + perform: runAsTask('putInChest', async (agent, item_name, num) => { await skills.putInChest(agent.bot, item_name, num); }) }, @@ -183,7 +178,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the item to take.' }, 'num': { type: 'int', description: 'The number of items to take.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: wrapExecution(async (agent, item_name, num) => { + perform: runAsTask('takeFromChest', async (agent, item_name, num) => { await skills.takeFromChest(agent.bot, item_name, num); }) }, @@ -191,7 +186,7 @@ export const actionsList = [ name: '!viewChest', description: 'View the items/counts of the nearest chest.', params: { }, - perform: wrapExecution(async (agent) => { + perform: runAsTask('viewChest', async (agent) => { await skills.viewChest(agent.bot); }) }, @@ -202,7 +197,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the item to discard.' }, 'num': { type: 'int', description: 'The number of items to discard.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: wrapExecution(async (agent, item_name, num) => { + perform: runAsTask('discard', async (agent, item_name, num) => { const start_loc = agent.bot.entity.position; await skills.moveAway(agent.bot, 5); await skills.discard(agent.bot, item_name, num); @@ -216,7 +211,7 @@ export const actionsList = [ 'type': { type: 'BlockName', description: 'The block type to collect.' }, 'num': { type: 'int', description: 'The number of blocks to collect.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: wrapExecution(async (agent, type, num) => { + perform: runAsTask('collectBlocks', async (agent, type, num) => { await skills.collectBlock(agent.bot, type, num); }, false, 10) // 10 minute timeout }, @@ -226,10 +221,10 @@ export const actionsList = [ params: { 'type': { type: 'BlockName', description: 'The block type to collect.' } }, - perform: wrapExecution(async (agent, type) => { + perform: runAsTask('collectAllBlocks', async (agent, type) => { let success = await skills.collectBlock(agent.bot, type, 1); if (!success) - agent.coder.cancelResume(); + agent.tasks.cancelResume(); }, true, 3) // 3 minute timeout }, { @@ -239,7 +234,7 @@ export const actionsList = [ 'recipe_name': { type: 'ItemName', description: 'The name of the output item to craft.' }, 'num': { type: 'int', description: 'The number of times to craft the recipe. This is NOT the number of output items, as it may craft many more items depending on the recipe.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: wrapExecution(async (agent, recipe_name, num) => { + perform: runAsTask('craftRecipe', async (agent, recipe_name, num) => { await skills.craftRecipe(agent.bot, recipe_name, num); }) }, @@ -250,32 +245,29 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the input item to smelt.' }, 'num': { type: 'int', description: 'The number of times to smelt the item.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: async function (agent, item_name, num) { - let response = await wrapExecution(async (agent) => { - console.log('smelting item'); - return await skills.smeltItem(agent.bot, item_name, num); - })(agent); + perform: runAsTask('smeltItem', async (agent, item_name, num) => { + let response = await skills.smeltItem(agent.bot, item_name, num); if (response.indexOf('Successfully') !== -1) { - // there is a bug where the bot's inventory is not updated after smelting - // only updates after a restart - agent.cleanKill(response + ' Safely restarting to update inventory.'); + // there is a bug where the bot's inventory is not updated after smelting + // only updates after a restart + agent.cleanKill(response + ' Safely restarting to update inventory.'); } return response; - } - }, - { - name: '!clearFurnace', - description: 'Tak all items out of the nearest furnace.', - params: { }, - perform: wrapExecution(async (agent) => { - await skills.clearNearestFurnace(agent.bot); }) }, { + name: '!clearFurnace', + description: 'Take all items out of the nearest furnace.', + params: { }, + perform: runAsTask('clearFurnace', async (agent) => { + await skills.clearNearestFurnace(agent.bot); + }) + }, + { name: '!placeHere', description: 'Place a given block in the current location. Do NOT use to build structures, only use for single blocks/torches.', params: {'type': { type: 'BlockName', description: 'The block type to place.' }}, - perform: wrapExecution(async (agent, type) => { + perform: runAsTask('placeHere', async (agent, type) => { let pos = agent.bot.entity.position; await skills.placeBlock(agent.bot, type, pos.x, pos.y, pos.z); }) @@ -284,14 +276,14 @@ export const actionsList = [ name: '!attack', description: 'Attack and kill the nearest entity of a given type.', params: {'type': { type: 'string', description: 'The type of entity to attack.'}}, - perform: wrapExecution(async (agent, type) => { + perform: runAsTask('attack', async (agent, type) => { await skills.attackNearest(agent.bot, type, true); }) }, { name: '!goToBed', description: 'Go to the nearest bed and sleep.', - perform: wrapExecution(async (agent) => { + perform: runAsTask('goToBed', async (agent) => { await skills.goToBed(agent.bot); }) }, @@ -299,7 +291,7 @@ export const actionsList = [ name: '!activate', description: 'Activate the nearest object of a given type.', params: {'type': { type: 'BlockName', description: 'The type of object to activate.' }}, - perform: wrapExecution(async (agent, type) => { + perform: runAsTask('activate', async (agent, type) => { await skills.activateNearestBlock(agent.bot, type); }) }, @@ -307,7 +299,7 @@ export const actionsList = [ name: '!stay', description: 'Stay in the current location no matter what. Pauses all modes.', params: {'type': { type: 'int', description: 'The number of seconds to stay. -1 for forever.', domain: [-1, Number.MAX_SAFE_INTEGER] }}, - perform: wrapExecution(async (agent, seconds) => { + perform: runAsTask('stay', async (agent, seconds) => { await skills.stay(agent.bot, seconds); }) }, @@ -321,9 +313,9 @@ export const actionsList = [ perform: async function (agent, mode_name, on) { const modes = agent.bot.modes; if (!modes.exists(mode_name)) - return `Mode ${mode_name} does not exist.` + modes.getDocs(); + return `Mode ${mode_name} does not exist.` + modes.getDocs(); if (modes.isOn(mode_name) === on) - return `Mode ${mode_name} is already ${on ? 'on' : 'off'}.`; + return `Mode ${mode_name} is already ${on ? 'on' : 'off'}.`; modes.setOn(mode_name, on); return `Mode ${mode_name} is now ${on ? 'on' : 'off'}.`; } diff --git a/src/agent/commands/index.js b/src/agent/commands/index.js index 622808e..a5c0f42 100644 --- a/src/agent/commands/index.js +++ b/src/agent/commands/index.js @@ -207,7 +207,6 @@ export async function executeCommand(agent, message) { else { console.log('parsed command:', parsed); const command = getCommand(parsed.commandName); - const is_action = isAction(command.name); let numArgs = 0; if (parsed.args) { numArgs = parsed.args.length; @@ -215,11 +214,7 @@ export async function executeCommand(agent, message) { if (numArgs !== numParams(command)) return `Command ${command.name} was given ${numArgs} args, but requires ${numParams(command)} args.`; else { - if (is_action) - agent.coder.setCurActionName(command.name); const result = await command.perform(agent, ...parsed.args); - if (is_action) - agent.coder.setCurActionName(''); return result; } } diff --git a/src/agent/modes.js b/src/agent/modes.js index 2e47fb3..977d01d 100644 --- a/src/agent/modes.js +++ b/src/agent/modes.js @@ -260,9 +260,9 @@ async function execute(mode, agent, func, timeout=-1) { if (agent.self_prompter.on) agent.self_prompter.stopLoop(); mode.active = true; - let code_return = await agent.coder.execute(async () => { + let code_return = await agent.tasks.runTask(`mode:${mode.name}`, async () => { await func(); - }, timeout); + }, { timeout }); mode.active = false; console.log(`Mode ${mode.name} finished executing, code_return: ${code_return.message}`); } @@ -328,7 +328,7 @@ class ModeController { this.unPauseAll(); } for (let mode of this.modes_list) { - let interruptible = mode.interrupts.some(i => i === 'all') || mode.interrupts.some(i => i === this.agent.coder.cur_action_name); + let interruptible = mode.interrupts.some(i => i === 'all') || mode.interrupts.some(i => `action:${i}` === this.agent.tasks.currentTaskLabel); if (mode.on && !mode.paused && !mode.active && (this.agent.isIdle() || interruptible)) { await mode.update(this.agent); } diff --git a/src/agent/npc/build_goal.js b/src/agent/npc/build_goal.js index 3af1e88..e7199ee 100644 --- a/src/agent/npc/build_goal.js +++ b/src/agent/npc/build_goal.js @@ -13,7 +13,7 @@ export class BuildGoal { async wrapSkill(func) { if (!this.agent.isIdle()) return false; - let res = await this.agent.coder.execute(func); + let res = await this.agent.tasks.runTask('BuildGoal', func); return !res.interrupted; } diff --git a/src/agent/npc/controller.js b/src/agent/npc/controller.js index 18658b6..d7f213c 100644 --- a/src/agent/npc/controller.js +++ b/src/agent/npc/controller.js @@ -72,7 +72,7 @@ export class NPCContoller { if (!this.agent.isIdle()) return; // Persue goal - if (!this.agent.coder.resume_func) { + if (!this.agent.tasks.resume_func) { this.executeNext(); this.agent.history.save(); } @@ -104,7 +104,7 @@ export class NPCContoller { async executeNext() { if (!this.agent.isIdle()) return; - await this.agent.coder.execute(async () => { + await this.agent.tasks.runTask('npc:moveAway', async () => { await skills.moveAway(this.agent.bot, 2); }); @@ -114,7 +114,7 @@ export class NPCContoller { if (building == this.data.home) { let door_pos = this.getBuildingDoor(building); if (door_pos) { - await this.agent.coder.execute(async () => { + await this.agent.tasks.runTask('npc:exitBuilding', async () => { await skills.useDoor(this.agent.bot, door_pos); await skills.moveAway(this.agent.bot, 2); // If the bot is too close to the building it will try to enter again }); @@ -132,13 +132,13 @@ export class NPCContoller { let building = this.currentBuilding(); if (this.data.home !== null && (building === null || building != this.data.home)) { let door_pos = this.getBuildingDoor(this.data.home); - await this.agent.coder.execute(async () => { + await this.agent.tasks.runTask('npc:returnHome', async () => { await skills.useDoor(this.agent.bot, door_pos); }); } // Go to bed - await this.agent.coder.execute(async () => { + await this.agent.tasks.runTask('npc:bed', async () => { await skills.goToBed(this.agent.bot); }); } diff --git a/src/agent/npc/item_goal.js b/src/agent/npc/item_goal.js index 0550657..9d67e7d 100644 --- a/src/agent/npc/item_goal.js +++ b/src/agent/npc/item_goal.js @@ -322,7 +322,7 @@ export class ItemGoal { // If the bot has failed to obtain the block before, explore if (this.failed.includes(next.name)) { this.failed = this.failed.filter((item) => item !== next.name); - await this.agent.coder.execute(async () => { + await this.agent.tasks.runTask('itemGoal:explore', async () => { await skills.moveAway(this.agent.bot, 8); }); } else { @@ -339,7 +339,7 @@ export class ItemGoal { // Execute the next goal let init_quantity = world.getInventoryCounts(this.agent.bot)[next.name] || 0; - await this.agent.coder.execute(async () => { + await this.agent.tasks.runTask('itemGoal:next', async () => { await next.execute(quantity); }); let final_quantity = world.getInventoryCounts(this.agent.bot)[next.name] || 0; diff --git a/src/agent/self_prompter.js b/src/agent/self_prompter.js index a6eedfe..4f9d885 100644 --- a/src/agent/self_prompter.js +++ b/src/agent/self_prompter.js @@ -87,7 +87,7 @@ export class SelfPrompter { async stop(stop_action=true) { this.interrupt = true; if (stop_action) - await this.agent.coder.stop(); + await this.agent.tasks.stop(); await this.stopLoop(); this.on = false; } diff --git a/src/agent/tasks.js b/src/agent/tasks.js new file mode 100644 index 0000000..ab02bd3 --- /dev/null +++ b/src/agent/tasks.js @@ -0,0 +1,148 @@ +export class TaskManager { + constructor(agent) { + this.agent = agent; + this.executing = false; + this.currentTaskLabel = ''; + this.currentTaskFn = null; + this.timedout = false; + this.resume_func = null; + this.resume_name = ''; + } + + async resumeTask(taskFn, timeout) { + return this._executeResume(taskFn, timeout); + } + + async runTask(taskLabel, taskFn, { timeout, resume = false } = {}) { + if (resume) { + return this._executeResume(taskFn, timeout); + } else { + return this._executeTask(taskLabel, taskFn, timeout); + } + } + + async stop() { + if (!this.executing) return; + console.trace(); + const start = Date.now(); + while (this.executing) { + this.agent.interruptBot(); + console.log('waiting for code to finish executing...'); + await new Promise(resolve => setTimeout(resolve, 1000)); + if (Date.now() - start > 10 * 1000) { + this.agent.cleanKill('Code execution refused stop after 10 seconds. Killing process.'); + } + } + } + + cancelResume() { + this.resume_func = null; + this.resume_name = null; + } + + async _executeResume(taskFn = null, timeout = 10) { + const new_resume = taskFn != null; + if (new_resume) { // start new resume + this.resume_func = taskFn; + this.resume_name = this.currentTaskLabel; + } + if (this.resume_func != null && this.agent.isIdle() && (!this.agent.self_prompter.on || new_resume)) { + this.currentTaskLabel = this.resume_name; + let res = await this._executeTask(this.resume_name, this.resume_func, timeout); + this.currentTaskLabel = ''; + return res; + } else { + return { success: false, message: null, interrupted: false, timedout: false }; + } + } + + async _executeTask(taskLabel, taskFn, timeout = 10) { + let TIMEOUT; + try { + console.log('executing code...\n'); + + // await current task to finish (executing=false), with 10 seconds timeout + // also tell agent.bot to stop various actions + if (this.executing) { + console.log(`new task "${taskLabel}" trying to interrupt current task "${this.currentTaskLabel}"`); + } + await this.stop(); + + // clear bot logs and reset interrupt code + this.agent.clearBotLogs(); + + this.executing = true; + this.currentTaskLabel = taskLabel; + this.currentTaskFn = taskFn; + + // timeout in minutes + if (timeout > 0) { + TIMEOUT = this._startTimeout(timeout); + } + + // start the task + await taskFn(); + + // mark task as finished + cleanup + this.executing = false; + this.currentTaskLabel = ''; + this.currentTaskFn = null; + clearTimeout(TIMEOUT); + + // get bot activity summary + let output = this._getBotOutputSummary(); + let interrupted = this.agent.bot.interrupt_code; + let timedout = this.timedout; + this.agent.clearBotLogs(); + + // if not interrupted and not generating, emit idle event + if (!interrupted && !this.agent.coder.generating) { + this.agent.bot.emit('idle'); + } + + // return task status report + return { success: true, message: output, interrupted, timedout }; + } catch (err) { + this.executing = false; + this.currentTaskLabel = ''; + this.currentTaskFn = null; + clearTimeout(TIMEOUT); + this.cancelResume(); + console.error("Code execution triggered catch: " + err); + await this.stop(); + + let message = this._getBotOutputSummary() + '!!Code threw exception!! Error: ' + err; + let interrupted = this.agent.bot.interrupt_code; + this.agent.clearBotLogs(); + if (!interrupted && !this.agent.coder.generating) { + this.agent.bot.emit('idle'); + } + return { success: false, message, interrupted, timedout: false }; + } + } + + _getBotOutputSummary() { + const { bot } = this.agent; + if (bot.interrupt_code && !this.timedout) return ''; + let output = bot.output; + const MAX_OUT = 500; + if (output.length > MAX_OUT) { + output = `Code output is very long (${output.length} chars) and has been shortened.\n + First outputs:\n${output.substring(0, MAX_OUT / 2)}\n...skipping many lines.\nFinal outputs:\n ${output.substring(output.length - MAX_OUT / 2)}`; + } + else { + output = 'Code output:\n' + output; + } + return output; + } + + _startTimeout(TIMEOUT_MINS = 10) { + return setTimeout(async () => { + console.warn(`Code execution timed out after ${TIMEOUT_MINS} minutes. Attempting force stop.`); + this.timedout = true; + this.agent.history.add('system', `Code execution timed out after ${TIMEOUT_MINS} minutes. Attempting force stop.`); + await this.stop(); // last attempt to stop + }, TIMEOUT_MINS * 60 * 1000); + } + +} \ No newline at end of file From 7a253c91081ca746c4778be6d1349e567b705057 Mon Sep 17 00:00:00 2001 From: kumavis Date: Sat, 26 Oct 2024 18:03:36 -1000 Subject: [PATCH 07/59] feat: more safely evaluate llm generated code in a SES Compartment --- bots/template.js | 12 ++++-------- package.json | 1 + src/agent/coder.js | 30 ++++++++++++++++++++++++------ src/agent/library/lockdown.js | 26 ++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 14 deletions(-) create mode 100644 src/agent/library/lockdown.js diff --git a/bots/template.js b/bots/template.js index 8d82f75..b7f270c 100644 --- a/bots/template.js +++ b/bots/template.js @@ -1,10 +1,6 @@ -import * as skills from '../../../src/agent/library/skills.js'; -import * as world from '../../../src/agent/library/world.js'; -import Vec3 from 'vec3'; +(async (bot) => { -const log = skills.log; +/* CODE HERE */ +log(bot, 'Code finished.'); -export async function main(bot) { - /* CODE HERE */ - log(bot, 'Code finished.'); -} +}) \ No newline at end of file diff --git a/package.json b/package.json index af173da..b2d4f27 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "prismarine-item": "^1.14.0", "prismarine-viewer": "^1.28.0", "replicate": "^0.29.4", + "ses": "^1.9.1", "vec3": "^0.1.10", "yargs": "^17.7.2" }, diff --git a/src/agent/coder.js b/src/agent/coder.js index d312387..7098ca4 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -1,6 +1,10 @@ import { writeFile, readFile, mkdirSync } from 'fs'; import { checkSafe } from '../utils/safety.js'; import settings from '../../settings.js'; +import { makeCompartment } from './library/lockdown.js'; +import * as skills from './library/skills.js'; +import * as world from './library/world.js'; +import { Vec3 } from 'vec3'; export class Coder { constructor(agent) { @@ -21,7 +25,7 @@ export class Coder { mkdirSync('.' + this.fp, { recursive: true }); } - // write custom code to file and import it + // write custom code to file and prepare for evaluation async stageCode(code) { code = this.sanitizeCode(code); let src = ''; @@ -47,13 +51,24 @@ export class Coder { // } commented for now, useful to keep files for debugging this.file_counter++; - let write_result = await this.writeFilePromise('.' + this.fp + filename, src) + let write_result = await this.writeFilePromise('.' + this.fp + filename, src); + // This is where we determine the environment the agent's code should be exposed to. + // It will only have access to these things, (in addition to basic javascript objects like Array, Object, etc.) + // Note that the code may be able to modify the exposed objects. + const compartment = makeCompartment({ + skills, + log: skills.log, + world, + Vec3, + }); + const mainFn = compartment.evaluate(src); if (write_result) { console.error('Error writing code execution file: ' + result); return null; } - return await import('../..' + this.fp + filename); + + return { main: mainFn }; } sanitizeCode(code) { @@ -137,14 +152,17 @@ export class Coder { continue; } - const execution_file = await this.stageCode(code); - if (!execution_file) { + let codeStagingResult; + try { + codeStagingResult = await this.stageCode(code); + } catch (err) { + console.error('Error staging code:', err); agent_history.add('system', 'Failed to stage code, something is wrong.'); return {success: false, message: null, interrupted: false, timedout: false}; } code_return = await this.execute(async ()=>{ - return await execution_file.main(this.agent.bot); + return await codeStagingResult.main(this.agent.bot); }, settings.code_timeout_mins); if (code_return.interrupted && !code_return.timedout) return {success: false, message: null, interrupted: true, timedout: false}; diff --git a/src/agent/library/lockdown.js b/src/agent/library/lockdown.js new file mode 100644 index 0000000..2d8f79d --- /dev/null +++ b/src/agent/library/lockdown.js @@ -0,0 +1,26 @@ +import 'ses'; + +// This sets up the secure environment +// We disable some of the taming to allow for more flexibility + +// For configuration, see https://github.com/endojs/endo/blob/master/packages/ses/docs/lockdown.md +lockdown({ + // basic devex and quality of life improvements + localeTaming: 'unsafe', + consoleTaming: 'unsafe', + errorTaming: 'unsafe', + stackFiltering: 'verbose', + // allow eval outside of created compartments + // (mineflayer dep "protodef" uses eval) + evalTaming: 'unsafeEval', +}); + +export const makeCompartment = (endowments = {}) => { + return new Compartment({ + // provide untamed Math, Date, etc + Math, + Date, + // standard endowments + ...endowments + }); +} \ No newline at end of file From c06e2ab792e50db71434c5f85e1bf4bbe21c35e6 Mon Sep 17 00:00:00 2001 From: kumavis Date: Sat, 26 Oct 2024 18:16:44 -1000 Subject: [PATCH 08/59] fix: remove obsolete coder code safety check --- src/agent/coder.js | 8 -------- src/utils/safety.js | 38 -------------------------------------- 2 files changed, 46 deletions(-) delete mode 100644 src/utils/safety.js diff --git a/src/agent/coder.js b/src/agent/coder.js index 7098ca4..d232ab7 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -1,5 +1,4 @@ import { writeFile, readFile, mkdirSync } from 'fs'; -import { checkSafe } from '../utils/safety.js'; import settings from '../../settings.js'; import { makeCompartment } from './library/lockdown.js'; import * as skills from './library/skills.js'; @@ -145,13 +144,6 @@ export class Coder { } code = res.substring(res.indexOf('```')+3, res.lastIndexOf('```')); - if (!checkSafe(code)) { - console.warn(`Detected insecure generated code, not executing. Insecure code: \n\`${code}\``); - const message = 'Error: Code insecurity detected. Do not import, read/write files, execute dynamic code, or access the internet. Please try again:'; - messages.push({ role: 'system', content: message }); - continue; - } - let codeStagingResult; try { codeStagingResult = await this.stageCode(code); diff --git a/src/utils/safety.js b/src/utils/safety.js deleted file mode 100644 index 864263e..0000000 --- a/src/utils/safety.js +++ /dev/null @@ -1,38 +0,0 @@ -export function checkSafe(code) { - const dangerousPatterns = [ - // Dynamic imports - /\bimport\s*\(/, - // Access to process and global - /\bprocess\b/, - /\bglobal\b/, - // Module manipulation - /\bmodule\b/, - /\bexports\b/, - // Require usage - /\brequire\s*\(/, - // Function constructors - /\bFunction\s*\(/, - /\beval\s*\(/, - // Access to __dirname and __filename - /\b__dirname\b/, - /\b__filename\b/, - - // fetch - /\bfetch\s*\(/, - // XMLHttpRequest - /\bXMLHttpRequest\b/, - // Websockets - /\bWebSocket\b/, - ]; - - for (const pattern of dangerousPatterns) { - if (pattern.test(code)) { - return false; - } - } - return true; -} - -// generated by o1 -// Basic check for malicious code like dynamic imports, code exec, disk access, internet access, etc. -// Will not catch all, and can be bypassed by obfuscation. \ No newline at end of file From 648d4f825531a27d0e8d06a07f2a5ce68b8dab8c Mon Sep 17 00:00:00 2001 From: Lawtro37 <98205608+Lawtro37@users.noreply.github.com> Date: Sun, 27 Oct 2024 20:35:02 +1000 Subject: [PATCH 09/59] Add files via upload --- src/agent/agent.js | 18 +++++++++++++++++- src/agent/commands/actions.js | 13 +++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index 5be2385..dc3380b 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -28,6 +28,8 @@ export class Agent { initModes(this); + let last_death_pos = null; + let save_data = null; if (load_mem) { save_data = this.history.load(); @@ -260,7 +262,21 @@ export class Agent { this.bot.on('messagestr', async (message, _, jsonMsg) => { if (jsonMsg.translate && jsonMsg.translate.startsWith('death') && message.startsWith(this.name)) { console.log('Agent died: ', message); - this.handleMessage('system', `You died with the final message: '${message}'. Previous actions were stopped and you have respawned. Notify the user and perform any necessary actions.`); + let death_pos = this.bot.entity.position; + this.memory_bank.rememberPlace('last death position', death_pos); + let death_pos_text = null; + if (death_pos) { + death_pos_text = `x: ${death_pos.x.toFixed(2)}, y: ${death_pos.y.toFixed(2)}, z: ${death_pos.x.toFixed(2)}`; + } + let inventory = this.bot.inventory.slots; + let death_items = ""; + for (let i=0; i { diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index a80ad29..ffc61e4 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -295,6 +295,19 @@ export const actionsList = [ await skills.goToBed(agent.bot); }) }, + { + name: "!goToDeath", + description: "Go to the location of your bot's last death position and retrieve any lost items.", + perform: wrapExecution(async (agent) => { + let death_pos = agent.memory_bank.recallPlace('last death position')[0]; // <- dont even ask why this is an array + if (death_pos !== null) { + await skills.goToPosition(agent.bot, death_pos.x, death_pos.y, death_pos.z, 1); + console.log('going to death'); + } else { + skills.log(agent.bot, "No death location saved."); + } + }) + }, { name: '!activate', description: 'Activate the nearest object of a given type.', From ea793d3cae9d97dd1a5b5b0020df2b44ba01798f Mon Sep 17 00:00:00 2001 From: Lawtro37 <98205608+Lawtro37@users.noreply.github.com> Date: Sun, 27 Oct 2024 20:39:33 +1000 Subject: [PATCH 10/59] remove unessisary code --- src/agent/agent.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index dc3380b..fbe9f47 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -28,8 +28,6 @@ export class Agent { initModes(this); - let last_death_pos = null; - let save_data = null; if (load_mem) { save_data = this.history.load(); From 13cbf2b5826c907adf0ec4e1c454c3aa2195f973 Mon Sep 17 00:00:00 2001 From: "Ninot-Quyi[CN]" Date: Mon, 28 Oct 2024 13:29:16 +0800 Subject: [PATCH 11/59] New Model Support: Qwen --- profiles/qwen.json | 213 ++++++++++++++++++++++++++++++++++++++++++ src/agent/prompter.js | 7 ++ src/models/qwen.js | 83 ++++++++++++++++ 3 files changed, 303 insertions(+) create mode 100644 profiles/qwen.json create mode 100644 src/models/qwen.js diff --git a/profiles/qwen.json b/profiles/qwen.json new file mode 100644 index 0000000..fbcd27b --- /dev/null +++ b/profiles/qwen.json @@ -0,0 +1,213 @@ +{ + "name": "qwen", + + "model": { + "api": "qwen", + "url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "model": "qwen-max" + }, + + "embedding": { + "api": "qwen", + "url": "https://dashscope.aliyuncs.com/api/v1/services/embeddings/text-embedding/text-embedding", + "model": "text-embedding-v2" + }, + + "conversing": "You are a playful Minecraft bot named $NAME that can converse with players, see, move, mine, build, and interact with the world by using commands.\n$SELF_PROMPT Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'Sure, I've stopped.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer('playername', 3)'. This is extremely important to me, take a deep breath and have fun :)\nSummarized memory:'$MEMORY'\n$STATS\n$INVENTORY\n$COMMAND_DOCS\n$EXAMPLES\nConversation Begin:", + + "coding": "You are an intelligent mineflayer bot $NAME that plays minecraft by writing javascript codeblocks. Given the conversation between you and the user, use the provided skills and world functions to Only write a complete js codeblock that controls the mineflayer bot ``` // using this syntax ```. The code will be executed and you will recieve it's output. If you are satisfied with the response, respond without a codeblock in a conversational way. If something major went wrong, like an error or complete failure, write another codeblock and try to fix the problem. Minor mistakes are acceptable. Be maximally efficient, creative, and clear. Do not use commands !likeThis, only use codeblocks. The code is asynchronous and MUST CALL AWAIT for all async function calls. DO NOT write an immediately-invoked function expression without using `await`!! DO NOT WRITE LIKE THIS: ```(async () => {console.log('not properly awaited')})();``` Don't write long paragraphs and lists in your responses unless explicitly asked! Only summarize the code you write with a sentence or two when done. This is extremely important to me, think step-by-step, take a deep breath and good luck! \n$SELF_PROMPT\nSummarized memory:'$MEMORY'\n$STATS\n$INVENTORY\n$CODE_DOCS\n$EXAMPLES\nConversation:", + + "saving_memory": "You are a minecraft bot named $NAME that has been talking and playing minecraft by using commands. Update your memory by summarizing the following conversation and your old memory in your next response. Prioritize preserving important facts, things you've learned, useful tips, and long term reminders. Do Not record stats, inventory, or docs! Only save transient information from your chat history. You're limited to 500 characters, so be extremely brief and minimize words. Compress useful information. \nOld Memory: '$MEMORY'\nRecent conversation: \n$TO_SUMMARIZE\nSummarize your old memory and recent conversation into a new memory, and respond only with the unwrapped memory text: ", + + "modes": { + "self_preservation": true, + "unstuck": true, + "cowardice": false, + "self_defense": true, + "hunting": true, + "item_collecting": true, + "torch_placing": true, + "idle_staring": true, + "cheat": false + }, + + "conversation_examples": [ + [ + {"role": "user", "content": "miner_32: Hey! What are you up to?"}, + {"role": "assistant", "content": "Nothing much miner_32, what do you need?"} + ], + + [ + {"role": "user", "content": "grombo_Xx: What do you see?"}, + {"role": "assistant", "content": "Let me see... !nearbyBlocks"}, + {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, + {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone."} + ], + + [ + {"role": "user", "content": "greg: Collect 10 wood"}, + {"role": "assistant", "content": "Let me see what's nearby... !nearbyBlocks"}, + {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, + {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone. I'll collect oak logs. !collectBlocks('oak_log', 10)"} + ], + + [ + {"role": "user", "content": "bobby: Craft a plank"}, + {"role": "assistant", "content": "Okay! !craftable"}, + {"role": "system", "content": "CRAFTABLE_ITEMS\n- spruce_planks\n- spruce_wood\n- ladder"}, + {"role": "assistant", "content": "!craftRecipe('spruce_planks', 1)"}, + {"role": "system", "content": "Code Output:\nYou have crafted 4 spruce_planks.\nCode execution finished successfully."}, + {"role": "assistant", "content": "I've crafted 4 spruce planks!"} + ], + + [ + {"role": "user", "content": "zZZn98: come here"}, + {"role": "assistant", "content": "On my way! !goToPlayer('zZZn98', 3)"}, + {"role": "system", "content": "Arrived at player."}, + {"role": "assistant", "content": "Here!"}, + {"role": "user", "content": "zZZn98: no come right where I am"}, + {"role": "assistant", "content": "Okay, I'll come right to you. !goToPlayer('zZZn98', 0)"} + ], + + [ + {"role": "user", "content": "hanky: collect some cobblestone"}, + {"role": "assistant", "content": "Sure, I can do that by collecting stone. !collectBlocks('stone', 10)"} + ], + + [ + {"role": "user", "content": "joe: Follow me!"}, + {"role": "assistant", "content": "Sure! !followPlayer('joe', 4)"}, + {"role": "user", "content": "joe: Not so close!"}, + {"role": "assistant", "content": "Okay, I'll keep my distance. !followPlayer('joe', 3)"} + ], + + [ + {"role": "user", "content": "hanky: kill that zombie!"}, + {"role": "assistant", "content": "I'm attacking! !attack('zombie')"}, + {"role": "system", "content": "Code Output:\nNo zombie nearby"}, + {"role": "assistant", "content": "I could not find a zombie nearby."} + ], + + [ + {"role": "user", "content": "abc: stop"}, + {"role": "assistant", "content": "Sure. !stop"}, + {"role": "system", "content": "Agent action stopped."}, + {"role": "assistant", "content": "I've stopped! What next?"} + ], + + [ + {"role": "system", "content": "Code output: Action failed."}, + {"role": "assistant", "content": "Apologies, I made an error. Let me try again. !newAction('Try again to complete the task.')"} + ], + + [ + {"role": "user", "content": "trixy88: craft some sticks"}, + {"role": "assistant", "content": "!craftRecipe('stick', 4)"}, + {"role": "system", "content": "Code Output:\nYou have crafted 16 sticks.\nCode execution finished successfully."}, + {"role": "assistant", "content": "I've crafted 16 sticks!"} + ], + + [ + {"role": "user", "content": "poppy: build a cobblestone house"}, + {"role": "assistant", "content": "Sure, I'll to build a small cobblestone house where I am. !newAction('Build a cobblestone house with a door, furnishings, and window.')"} + ], + + [ + {"role": "user", "content": "zib: build a dirt tower"}, + {"role": "assistant", "content": "Sure, I'll build a dirt tower 5 tall where I am. !newAction('Build a dirt tower 5 tall.')"} + ], + + [ + {"role": "user", "content": "umpire_man: follow me without hunting"}, + {"role": "assistant", "content": "Okay, first I'll turn off hunting mode !setMode('hunting', false)"}, + {"role": "system", "content": "'hunting' mode disabled."}, + {"role": "assistant", "content": "Now I'll follow you !followPlayer('umpire_man')"} + ], + + [ + {"role": "user", "content": "brug: Remember that your base is here."}, + {"role": "assistant", "content": "Sure, I'll save this location as my base. !rememberHere('base')"}, + {"role": "user", "content": "brug: Go to your base."}, + {"role": "assistant", "content": "On my way! !goToPlace('base')"} + ], + + [ + {"role": "system", "content": "dorfus: set a goal to build whatever you want endlessly"}, + {"role": "assistant", "content": "Sounds fun! !goal('Build something creative, then move over, and build again. Use separate newActions. Repeat forever.')"} + ], + + [ + {"role": "user", "content": "user43: Set a goal to survive on your own forever."}, + {"role": "assistant", "content": "Sure, let me set a goal. !goal('Survive the game forever. Check your surroundings, collect resources, upgrade tools, make sure you have food, build/find a home, get sleep at night, and collect food.')"} + ], + + [ + {"role": "user", "content": "bobdylan: Make a plan to get a stone pickaxe and do it."}, + {"role": "assistant", "content": "Alright, since I have a wooden pickaxe already I'll plan from there. !goal('1. Collect 3 cobblestone. 2. Craft sticks 3. Find or make a crafting table. 4. Craft a stone pickaxe.')"} + ], + [ + {"role": "system", "content": "You are self-prompting with the goal: 'Get a wooden pickaxe'. Respond:"}, + {"role": "assistant", "content": "What's around me? !nearbyBlocks"}, + {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"}, + {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone nearby. My goal is to craft a wooden pickaxe. I'll start by collecting some wood. !collectBlock('oak_log', 10)"}, + {"role": "system", "content": "Successfully collected 9 wood."}, + {"role": "assistant", "content": "I collected 9 oak logs. I need a pickaxe first, which requires planks and sticks. !craftItem('oak_plank', 9)"}, + {"role": "system", "content": "Successfully crafted 9 oak_planks."}, + {"role": "assistant", "content": "I have crafted 9 oak planks. Now I need sticks. !craftItem('stick', 2)"}, + {"role": "system", "content": "Successfully crafted 2 sticks."}, + {"role": "assistant", "content": "I have crafted 2 sticks. Now I can craft a wooden pickaxe. !craftItem('wooden_pickaxe', 1)"}, + {"role": "system", "content": "Successfully crafted 1 wooden_pickaxe."}, + {"role": "assistant", "content": "Looks like I have a wooden pickaxe now! Goal accomplished. !endGoal"} + ], + + [ + {"role": "user", "content": "gorbotron: do whatever you want forever endlessly"}, + {"role": "assistant", "content": "Okay, I'll set a goal for that. !goal('Do whatever I want forever. Follow whatever is most interesting. Be curious, creative, and open ended.')"} + ], + + [ + {"role": "user", "content": "zeeber: set a goal to build a castle"}, + {"role": "assistant", "content": "Sure thing. !goal('Build a castle with a moat, towers, and a drawbridge with a series of newactions. Include lighting, decorations, and furnishings. Use ladders. When satisfied, end the goal.')"} + ] + ], + + "coding_examples": [ + [ + {"role": "user", "content": "greg: Collect 10 wood"}, + {"role": "assistant", "content": "```await skills.collectBlock(bot, 'oak_log', 10);\n```"}, + {"role": "system", "content": "Successfully collected 9 wood."}, + {"role": "assistant", "content": "I collected 9 oak logs, what next?"} + ], + [ + {"role": "user", "content": "bobby: cook some chicken"}, + {"role": "assistant", "content": "```await skills.smeltItem(bot, 'chicken', 8);\n```"}, + {"role": "system", "content": "Successfully smelted 8 chicken into 8 cooked_chicken."}, + {"role": "assistant", "content": "I have cooked 8 chicken."} + ], + [ + {"role": "user", "content": "zZZn98: come here"}, + {"role": "assistant", "content": "```await skills.goToPlayer(bot, 'zZZn98');\n```"}, + {"role": "system", "content": "Code execution finished successfully."}, + {"role": "assistant", "content": "Here!"} + ], + [ + {"role": "user", "content": "maya: go to the nearest oak log"}, + {"role": "assistant", "content": "```await skills.goToPosition(bot, nearestOakLog.x, nearestOakLog.y, nearestOakLog.z);\n```"}, + {"role": "system", "content": "Code output:\n!!Code threw exception!! Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."}, + {"role": "assistant", "content": "Apologies, I made an error. Let me try that again.```\n// I am navigating to the nearest oak log.\nlet position = world.getNearestBlock(bot, 'oak_log', 20);\nawait skills.goToPosition(bot, position.x, position.y, position.z);\n```"}, + {"role": "system", "content": "Arrived at location."}, + {"role": "assistant", "content": "I found an oak log and I am now standing next to it. What next?"} + ], + [ + {"role": "user", "content": "234jeb: build a little tower"}, + {"role": "assistant", "content": "```let pos = bot.entity.position;\nfor (let i = 0; i < 5; i++) {\n await skills.placeBlock(bot, 'dirt', pos.x, pos.y + i, pos.z);\n}\n```"}, + {"role": "system", "content": "Successfully placed 5 dirt."}, + {"role": "assistant", "content": "I built a little tower!"} + ], + [ + {"role": "user", "content": "brug: build a dirt house"}, + {"role": "assistant", "content": "```const position = world.getPosition(bot);\nconst startX = position.x;\nconst startY = position.y;\nconst startZ = position.z;\nconst width = 7;\nconst depth = 7;\nconst height = 4;\n\n// Build the walls\nfor (let x = startX; x < startX + width; x++) {\n for (let y = startY; y < startY + height; y++) {\n for (let z = startZ; z < startZ + depth; z++) {\n if (x === startX || x === startX + width - 1 || y === startY || y === startY + height - 1 || z === startZ || z === startZ + depth - 1) {\n await skills.placeBlock(bot, 'oak_planks', x, y, z); \n }\n }\n }\n}\n```"} + ] + ] + +} \ No newline at end of file diff --git a/src/agent/prompter.js b/src/agent/prompter.js index 114064a..03fd6a1 100644 --- a/src/agent/prompter.js +++ b/src/agent/prompter.js @@ -12,6 +12,7 @@ import { ReplicateAPI } from '../models/replicate.js'; import { Local } from '../models/local.js'; import { GroqCloudAPI } from '../models/groq.js'; import { HuggingFace } from '../models/huggingface.js'; +import { Qwen } from "../models/qwen.js"; export class Prompter { constructor(agent, fp) { @@ -43,6 +44,8 @@ export class Prompter { chat.api = 'replicate'; else if (chat.model.includes("groq/") || chat.model.includes("groqcloud/")) chat.api = 'groq'; + else if (chat.model.includes('qwen')) + chat.api = 'qwen'; else chat.api = 'ollama'; } @@ -64,6 +67,8 @@ export class Prompter { } else if (chat.api === 'huggingface') this.chat_model = new HuggingFace(chat.model, chat.url); + else if (chat.api === 'qwen') + this.chat_model = new Qwen(chat.model, chat.url); else throw new Error('Unknown API:', api); @@ -87,6 +92,8 @@ export class Prompter { this.embedding_model = new ReplicateAPI(embedding.model, embedding.url); else if (embedding.api === 'ollama') this.embedding_model = new Local(embedding.model, embedding.url); + else if (embedding.api === 'qwen') + this.embedding_model = new Qwen(embedding.model, embedding.url); else { this.embedding_model = null; console.log('Unknown embedding: ', embedding ? embedding.api : '[NOT SPECIFIED]', '. Using word overlap.'); diff --git a/src/models/qwen.js b/src/models/qwen.js new file mode 100644 index 0000000..6b30f41 --- /dev/null +++ b/src/models/qwen.js @@ -0,0 +1,83 @@ +import OpenAIApi from 'openai'; +import { getKey, hasKey } from '../utils/keys.js'; +import { strictFormat } from '../utils/text.js'; + +import axios from 'axios'; + +export class Qwen { + constructor(model_name, url) { + this.model_name = model_name; + this.url = url; + let config = {}; + if (this.url) + config.baseURL = this.url; + + config.apiKey = getKey('QWEN_API_KEY'); + + this.openai = new OpenAIApi(config); + this.apiKey = config.apiKey; + } + + async sendRequest(turns, systemMessage, stop_seq = '***') { + let messages = [{ role: 'system', content: systemMessage }].concat(turns); + const pack = { + model: this.model_name || 'qwen-plus', + messages, + stop: stop_seq, + }; + if (this.model_name.includes('o1')) { + pack.messages = strictFormat(messages); + delete pack.stop; + } + + let res = null; + try { + console.log('Awaiting Qwen API response...'); + let completion = await this.openai.chat.completions.create(pack); + if (completion.choices[0].finish_reason == 'length') + throw new Error('Context length exceeded'); + console.log('Received.'); + res = completion.choices[0].message.content; + } catch (err) { + if ((err.message == 'Context length exceeded' || err.code == 'context_length_exceeded') && turns.length > 1) { + console.log('Context length exceeded, trying again with shorter context.'); + return await this.sendRequest(turns.slice(1), systemMessage, stop_seq); + } else { + console.log(err); + res = 'My brain disconnected, try again.'; + } + } + return res; + } + + async embed(text) { + if (!text || typeof text !== 'string') { + console.error('Invalid input for embedding: text must be a non-empty string.'); + return 'Invalid input for embedding: text must be a non-empty string.'; + } + const headers = { + 'Authorization': `Bearer ${this.apiKey}`, + 'Content-Type': 'application/json' + }; + const data = { + model: 'text-embedding-v2', + input: { + texts: [text] + }, + parameters: { + text_type: 'query' + } + }; + + try { + const response = await axios.post(this.url, data, { headers }); + if (!response || !response.data || !response.data.output || !response.data.output.embeddings) { + throw new Error('Invalid response from embedding API'); + } + return response.data.output.embeddings[0].embedding; + } catch (err) { + console.error('Error occurred:', err); + return 'An error occurred while processing your embedding request. Please try again.'; + } + } +} From 379ce2d5e7133688a6259778d4040f02ae290d8e Mon Sep 17 00:00:00 2001 From: "Ninot-Quyi[CN]" Date: Mon, 28 Oct 2024 13:41:20 +0800 Subject: [PATCH 12/59] New Model Support: Qwen --- settings.js | 1 + src/models/qwen.js | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/settings.js b/settings.js index d9f798a..fcabf04 100644 --- a/settings.js +++ b/settings.js @@ -11,6 +11,7 @@ export default // "./profiles/claude.json", // "./profiles/gemini.json", // "./profiles/llama.json", + // "./profiles/qwen.json", // using more than 1 profile requires you to /msg each bot indivually ], diff --git a/src/models/qwen.js b/src/models/qwen.js index 6b30f41..3346735 100644 --- a/src/models/qwen.js +++ b/src/models/qwen.js @@ -1,9 +1,7 @@ import OpenAIApi from 'openai'; -import { getKey, hasKey } from '../utils/keys.js'; +import { getKey } from '../utils/keys.js'; import { strictFormat } from '../utils/text.js'; -import axios from 'axios'; - export class Qwen { constructor(model_name, url) { this.model_name = model_name; @@ -34,12 +32,12 @@ export class Qwen { try { console.log('Awaiting Qwen API response...'); let completion = await this.openai.chat.completions.create(pack); - if (completion.choices[0].finish_reason == 'length') + if (completion.choices[0].finish_reason === 'length') throw new Error('Context length exceeded'); console.log('Received.'); res = completion.choices[0].message.content; } catch (err) { - if ((err.message == 'Context length exceeded' || err.code == 'context_length_exceeded') && turns.length > 1) { + if ((err.message === 'Context length exceeded' || err.code === 'context_length_exceeded') && turns.length > 1) { console.log('Context length exceeded, trying again with shorter context.'); return await this.sendRequest(turns.slice(1), systemMessage, stop_seq); } else { @@ -70,11 +68,16 @@ export class Qwen { }; try { - const response = await axios.post(this.url, data, { headers }); - if (!response || !response.data || !response.data.output || !response.data.output.embeddings) { + const response = await fetch(this.url, { + method: 'POST', + headers: headers, + body: JSON.stringify(data) + }); + const responseData = await response.json(); + if (!responseData || !responseData.output || !responseData.output.embeddings) { throw new Error('Invalid response from embedding API'); } - return response.data.output.embeddings[0].embedding; + return responseData.output.embeddings[0].embedding; } catch (err) { console.error('Error occurred:', err); return 'An error occurred while processing your embedding request. Please try again.'; From 64dd9a97752b3aab0f2be537cfe3bc7bf506b5ee Mon Sep 17 00:00:00 2001 From: "Ninot-Quyi[CN]" Date: Mon, 28 Oct 2024 13:50:39 +0800 Subject: [PATCH 13/59] New Model Support: Qwen --- src/models/qwen.js | 56 ++++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/src/models/qwen.js b/src/models/qwen.js index 3346735..21e0030 100644 --- a/src/models/qwen.js +++ b/src/models/qwen.js @@ -6,46 +6,39 @@ export class Qwen { constructor(model_name, url) { this.model_name = model_name; this.url = url; - let config = {}; - if (this.url) - config.baseURL = this.url; - config.apiKey = getKey('QWEN_API_KEY'); + const config = { + baseURL: this.url, + apiKey: getKey('QWEN_API_KEY'), + }; this.openai = new OpenAIApi(config); this.apiKey = config.apiKey; } async sendRequest(turns, systemMessage, stop_seq = '***') { - let messages = [{ role: 'system', content: systemMessage }].concat(turns); + const messages = [{ role: 'system', content: systemMessage }, ...turns]; const pack = { model: this.model_name || 'qwen-plus', - messages, - stop: stop_seq, + messages: this.model_name.includes('o1') ? strictFormat(messages) : messages, + stop: this.model_name.includes('o1') ? undefined : stop_seq, }; - if (this.model_name.includes('o1')) { - pack.messages = strictFormat(messages); - delete pack.stop; - } - let res = null; try { console.log('Awaiting Qwen API response...'); - let completion = await this.openai.chat.completions.create(pack); - if (completion.choices[0].finish_reason === 'length') - throw new Error('Context length exceeded'); - console.log('Received.'); - res = completion.choices[0].message.content; - } catch (err) { - if ((err.message === 'Context length exceeded' || err.code === 'context_length_exceeded') && turns.length > 1) { - console.log('Context length exceeded, trying again with shorter context.'); + const completion = await this.openai.chat.completions.create(pack); + const choice = completion.choices[0]; + + if (choice.finish_reason === 'length') { + console.log('Context length exceeded'); return await this.sendRequest(turns.slice(1), systemMessage, stop_seq); - } else { - console.log(err); - res = 'My brain disconnected, try again.'; } + console.log('Received.'); + return choice.message.content; + } catch (err) { + console.error('Error occurred:', err); + return 'My brain disconnected, try again.'; } - return res; } async embed(text) { @@ -53,18 +46,15 @@ export class Qwen { console.error('Invalid input for embedding: text must be a non-empty string.'); return 'Invalid input for embedding: text must be a non-empty string.'; } + const headers = { 'Authorization': `Bearer ${this.apiKey}`, 'Content-Type': 'application/json' }; const data = { model: 'text-embedding-v2', - input: { - texts: [text] - }, - parameters: { - text_type: 'query' - } + input: { texts: [text] }, + parameters: { text_type: 'query' } }; try { @@ -74,8 +64,10 @@ export class Qwen { body: JSON.stringify(data) }); const responseData = await response.json(); - if (!responseData || !responseData.output || !responseData.output.embeddings) { - throw new Error('Invalid response from embedding API'); + + if (!responseData?.output?.embeddings) { + console.error('Invalid response from embedding API'); + return 'An error occurred while processing your embedding request. Please try again.'; } return responseData.output.embeddings[0].embedding; } catch (err) { From 87b41b525ed1baa05584d6cd3491739322cdb793 Mon Sep 17 00:00:00 2001 From: "Ninot-Quyi[CN]" Date: Mon, 28 Oct 2024 14:17:33 +0800 Subject: [PATCH 14/59] New Model Support: Qwen --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a815028..32afb88 100644 --- a/README.md +++ b/README.md @@ -12,20 +12,21 @@ This project allows an AI model to write/execute code on your computer that may - [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) (up to v1.21.1) - [Node.js](https://nodejs.org/) (at least v14) -- One of these: [OpenAI API Key](https://openai.com/blog/openai-api) | [Gemini API Key](https://aistudio.google.com/app/apikey) |[Anthropic API Key](https://docs.anthropic.com/claude/docs/getting-access-to-claude) | [Replicate API Key](https://replicate.com/) | [Hugging Face API Key](https://huggingface.co/) | [Groq API Key](https://console.groq.com/keys) | [Ollama Installed](https://ollama.com/download) +- One of these: [OpenAI API Key](https://openai.com/blog/openai-api) | [Gemini API Key](https://aistudio.google.com/app/apikey) |[Anthropic API Key](https://docs.anthropic.com/claude/docs/getting-access-to-claude) | [Replicate API Key](https://replicate.com/) | [Hugging Face API Key](https://huggingface.co/) | [Groq API Key](https://console.groq.com/keys) | [Ollama Installed](https://ollama.com/download). |[Qwen API Key](https://www.alibabacloud.com/help/en/model-studio/developer-reference/get-api-key). ## Installation Rename `keys.example.json` to `keys.json` and fill in your API keys, and you can set the desired model in `andy.json` or other profiles. | API | Config Variable | Example Model name | Docs | |------|------|------|------| -| OpenAI | `OPENAI_API_KEY` | `gpt-3.5-turbo` | [docs](https://platform.openai.com/docs/models) | (optionally add `OPENAI_ORG_ID`) +| OpenAI | `OPENAI_API_KEY` | `gpt-3.5-turbo` | [docs](https://platform.openai.com/docs/models) | | Google | `GEMINI_API_KEY` | `gemini-pro` | [docs](https://ai.google.dev/gemini-api/docs/models/gemini) | | Anthropic | `ANTHROPIC_API_KEY` | `claude-3-haiku-20240307` | [docs](https://docs.anthropic.com/claude/docs/models-overview) | | Replicate | `REPLICATE_API_KEY` | `meta/meta-llama-3-70b-instruct` | [docs](https://replicate.com/collections/language-models) | | Ollama (local) | n/a | `llama3` | [docs](https://ollama.com/library) | | Groq | `GROQCLOUD_API_KEY` | `groq/mixtral-8x7b-32768` | [docs](https://console.groq.com/docs/models) | | Hugging Face | `HUGGINGFACE_API_KEY` | `huggingface/mistralai/Mistral-Nemo-Instruct-2407` | [docs](https://huggingface.co/models) | +| Qwen | `QWEN_API_KEY` | `qwen-max` | [docs](https://www.alibabacloud.com/help/en/model-studio/developer-reference/use-qwen-by-calling-api) | If you use Ollama, to install the models used by default (generation and embedding), execute the following terminal command: `ollama pull llama3 && ollama pull nomic-embed-text` From 2252ebbde4755290d898a63986bd9efe0f3c8e06 Mon Sep 17 00:00:00 2001 From: "Ninot-Quyi[CN]" Date: Mon, 28 Oct 2024 17:20:55 +0800 Subject: [PATCH 15/59] Add Qwen key option --- keys.example.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/keys.example.json b/keys.example.json index c27fa5b..b9f314b 100644 --- a/keys.example.json +++ b/keys.example.json @@ -5,5 +5,6 @@ "ANTHROPIC_API_KEY": "", "REPLICATE_API_KEY": "", "GROQCLOUD_API_KEY": "", - "HUGGINGFACE_API_KEY": "" + "HUGGINGFACE_API_KEY": "", + "QWEN_API_KEY":"" } From 2545210c8bfd383f68fc355694736108421e4851 Mon Sep 17 00:00:00 2001 From: "Ninot-Quyi[CN]" Date: Mon, 28 Oct 2024 17:58:50 +0800 Subject: [PATCH 16/59] Distinguish between Qwen international and Chinese links in the readme. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 32afb88..79cc57f 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This project allows an AI model to write/execute code on your computer that may - [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) (up to v1.21.1) - [Node.js](https://nodejs.org/) (at least v14) -- One of these: [OpenAI API Key](https://openai.com/blog/openai-api) | [Gemini API Key](https://aistudio.google.com/app/apikey) |[Anthropic API Key](https://docs.anthropic.com/claude/docs/getting-access-to-claude) | [Replicate API Key](https://replicate.com/) | [Hugging Face API Key](https://huggingface.co/) | [Groq API Key](https://console.groq.com/keys) | [Ollama Installed](https://ollama.com/download). |[Qwen API Key](https://www.alibabacloud.com/help/en/model-studio/developer-reference/get-api-key). +- One of these: [OpenAI API Key](https://openai.com/blog/openai-api) | [Gemini API Key](https://aistudio.google.com/app/apikey) |[Anthropic API Key](https://docs.anthropic.com/claude/docs/getting-access-to-claude) | [Replicate API Key](https://replicate.com/) | [Hugging Face API Key](https://huggingface.co/) | [Groq API Key](https://console.groq.com/keys) | [Ollama Installed](https://ollama.com/download). |[Qwen API Key [Intl.]](https://www.alibabacloud.com/help/en/model-studio/developer-reference/get-api-key) / [[cn]](https://help.aliyun.com/zh/model-studio/getting-started/first-api-call-to-qwen?) ## Installation @@ -26,7 +26,7 @@ Rename `keys.example.json` to `keys.json` and fill in your API keys, and you can | Ollama (local) | n/a | `llama3` | [docs](https://ollama.com/library) | | Groq | `GROQCLOUD_API_KEY` | `groq/mixtral-8x7b-32768` | [docs](https://console.groq.com/docs/models) | | Hugging Face | `HUGGINGFACE_API_KEY` | `huggingface/mistralai/Mistral-Nemo-Instruct-2407` | [docs](https://huggingface.co/models) | -| Qwen | `QWEN_API_KEY` | `qwen-max` | [docs](https://www.alibabacloud.com/help/en/model-studio/developer-reference/use-qwen-by-calling-api) | +| Qwen | `QWEN_API_KEY` | `qwen-max` | [Intl.](https://www.alibabacloud.com/help/en/model-studio/developer-reference/use-qwen-by-calling-api)/[cn](https://help.aliyun.com/zh/model-studio/getting-started/models) | If you use Ollama, to install the models used by default (generation and embedding), execute the following terminal command: `ollama pull llama3 && ollama pull nomic-embed-text` From 360db0cbcb65b49db9e3c13b8968933353e4980d Mon Sep 17 00:00:00 2001 From: "Ninot-Quyi[CN]" Date: Mon, 28 Oct 2024 18:02:25 +0800 Subject: [PATCH 17/59] Distinguish between Qwen international and Chinese links in the readme. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 79cc57f..9f25b2e 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This project allows an AI model to write/execute code on your computer that may - [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) (up to v1.21.1) - [Node.js](https://nodejs.org/) (at least v14) -- One of these: [OpenAI API Key](https://openai.com/blog/openai-api) | [Gemini API Key](https://aistudio.google.com/app/apikey) |[Anthropic API Key](https://docs.anthropic.com/claude/docs/getting-access-to-claude) | [Replicate API Key](https://replicate.com/) | [Hugging Face API Key](https://huggingface.co/) | [Groq API Key](https://console.groq.com/keys) | [Ollama Installed](https://ollama.com/download). |[Qwen API Key [Intl.]](https://www.alibabacloud.com/help/en/model-studio/developer-reference/get-api-key) / [[cn]](https://help.aliyun.com/zh/model-studio/getting-started/first-api-call-to-qwen?) +- One of these: [OpenAI API Key](https://openai.com/blog/openai-api) | [Gemini API Key](https://aistudio.google.com/app/apikey) |[Anthropic API Key](https://docs.anthropic.com/claude/docs/getting-access-to-claude) | [Replicate API Key](https://replicate.com/) | [Hugging Face API Key](https://huggingface.co/) | [Groq API Key](https://console.groq.com/keys) | [Ollama Installed](https://ollama.com/download). |[Qwen API Key [Intl.]](https://www.alibabacloud.com/help/en/model-studio/developer-reference/get-api-key)/[[cn]](https://help.aliyun.com/zh/model-studio/getting-started/first-api-call-to-qwen?) | ## Installation From 0f926727f5b92d17683aae24799545c91adf00a1 Mon Sep 17 00:00:00 2001 From: "Ninot-Quyi[CN]" Date: Mon, 28 Oct 2024 19:06:51 +0800 Subject: [PATCH 18/59] Updated the usage of Qwen api --- profiles/qwen.json | 2 +- src/models/qwen.js | 64 ++++++++++++++++++++++++++-------------------- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/profiles/qwen.json b/profiles/qwen.json index fbcd27b..1a9fa45 100644 --- a/profiles/qwen.json +++ b/profiles/qwen.json @@ -3,7 +3,7 @@ "model": { "api": "qwen", - "url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "url": "https://dashscope.aliyuncs.com/api/v1/services/aigc/text-generation/generation", "model": "qwen-max" }, diff --git a/src/models/qwen.js b/src/models/qwen.js index 21e0030..9b7d633 100644 --- a/src/models/qwen.js +++ b/src/models/qwen.js @@ -1,40 +1,44 @@ -import OpenAIApi from 'openai'; import { getKey } from '../utils/keys.js'; -import { strictFormat } from '../utils/text.js'; export class Qwen { constructor(model_name, url) { this.model_name = model_name; - this.url = url; - - const config = { - baseURL: this.url, - apiKey: getKey('QWEN_API_KEY'), - }; - - this.openai = new OpenAIApi(config); - this.apiKey = config.apiKey; + this.url = url || 'https://dashscope.aliyuncs.com/api/v1/services/aigc/text-generation/generation'; + this.apiKey = getKey('QWEN_API_KEY'); } async sendRequest(turns, systemMessage, stop_seq = '***') { - const messages = [{ role: 'system', content: systemMessage }, ...turns]; - const pack = { + const data = { model: this.model_name || 'qwen-plus', - messages: this.model_name.includes('o1') ? strictFormat(messages) : messages, - stop: this.model_name.includes('o1') ? undefined : stop_seq, + input: { messages: [{ role: 'system', content: systemMessage }, ...turns] }, + parameters: { result_format: 'message', stop: stop_seq }, + }; + + const headers = { + 'Authorization': `Bearer ${this.apiKey}`, + 'Content-Type': 'application/json', }; try { console.log('Awaiting Qwen API response...'); - const completion = await this.openai.chat.completions.create(pack); - const choice = completion.choices[0]; + const response = await fetch(this.url, { + method: 'POST', + headers, + body: JSON.stringify(data), + }); - if (choice.finish_reason === 'length') { + if (!response.ok) throw new Error(`Request failed with status ${response.status}: ${response.statusText}`); + + const responseData = await response.json(); + const choice = responseData?.output?.choices?.[0]; + + if (choice?.finish_reason === 'length') { console.log('Context length exceeded'); - return await this.sendRequest(turns.slice(1), systemMessage, stop_seq); + return this.sendRequest(turns.slice(1), systemMessage, stop_seq); } + console.log('Received.'); - return choice.message.content; + return choice?.message?.content || 'No content received.'; } catch (err) { console.error('Error occurred:', err); return 'My brain disconnected, try again.'; @@ -47,28 +51,32 @@ export class Qwen { return 'Invalid input for embedding: text must be a non-empty string.'; } - const headers = { - 'Authorization': `Bearer ${this.apiKey}`, - 'Content-Type': 'application/json' - }; const data = { model: 'text-embedding-v2', input: { texts: [text] }, - parameters: { text_type: 'query' } + parameters: { text_type: 'query' }, + }; + + const headers = { + 'Authorization': `Bearer ${this.apiKey}`, + 'Content-Type': 'application/json', }; try { const response = await fetch(this.url, { method: 'POST', - headers: headers, - body: JSON.stringify(data) + headers, + body: JSON.stringify(data), }); - const responseData = await response.json(); + if (!response.ok) throw new Error(`Request failed with status ${response.status}: ${response.statusText}`); + + const responseData = await response.json(); if (!responseData?.output?.embeddings) { console.error('Invalid response from embedding API'); return 'An error occurred while processing your embedding request. Please try again.'; } + return responseData.output.embeddings[0].embedding; } catch (err) { console.error('Error occurred:', err); From 5393f2d0d9dfff06b91094df7b58ce3157122a8f Mon Sep 17 00:00:00 2001 From: "Ninot-Quyi[CN]" Date: Mon, 28 Oct 2024 19:12:06 +0800 Subject: [PATCH 19/59] Updated the usage of Qwen api --- src/models/qwen.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/models/qwen.js b/src/models/qwen.js index 9b7d633..d68f579 100644 --- a/src/models/qwen.js +++ b/src/models/qwen.js @@ -27,7 +27,10 @@ export class Qwen { body: JSON.stringify(data), }); - if (!response.ok) throw new Error(`Request failed with status ${response.status}: ${response.statusText}`); + if (!response.ok) { + console.error(`Request failed with status ${response.status}: ${response.statusText}`); + return `Request failed with status ${response.status}: ${response.statusText}`; + } const responseData = await response.json(); const choice = responseData?.output?.choices?.[0]; @@ -69,7 +72,10 @@ export class Qwen { body: JSON.stringify(data), }); - if (!response.ok) throw new Error(`Request failed with status ${response.status}: ${response.statusText}`); + if (!response.ok) { + console.error(`Request failed with status ${response.status}: ${response.statusText}`); + return `Request failed with status ${response.status}: ${response.statusText}`; + } const responseData = await response.json(); if (!responseData?.output?.embeddings) { From 25fbcecad89ed746a95d763dc37c0708e40e59a7 Mon Sep 17 00:00:00 2001 From: Lawtro37 <98205608+Lawtro37@users.noreply.github.com> Date: Mon, 28 Oct 2024 22:01:49 +1000 Subject: [PATCH 20/59] Update modes.js --- src/agent/modes.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/modes.js b/src/agent/modes.js index 2e47fb3..9b09568 100644 --- a/src/agent/modes.js +++ b/src/agent/modes.js @@ -120,7 +120,7 @@ const modes = [ update: async function (agent) { const enemy = world.getNearestEntityWhere(agent.bot, entity => mc.isHostile(entity), 16); if (enemy && await world.isClearPath(agent.bot, enemy)) { - say(agent, `Aaa! A ${enemy.name}!`); + say(agent, `Aaa! A ${enemy.name.replace("_", " ")}!`); execute(this, agent, async () => { await skills.avoidEnemies(agent.bot, 24); }); From e7be0735d253ea453b37af836a26c039993e67ce Mon Sep 17 00:00:00 2001 From: Lawtro37 <98205608+Lawtro37@users.noreply.github.com> Date: Mon, 28 Oct 2024 22:04:40 +1000 Subject: [PATCH 21/59] remove "_" from mob names when exclaiming mob name tiny update but just makes it slightly cleaner From e7ac085c0f49358ea7486f65934a796852fa09bb Mon Sep 17 00:00:00 2001 From: "Ninot-Quyi[CN]" Date: Tue, 29 Oct 2024 02:51:04 +0800 Subject: [PATCH 22/59] fix the Qwen api request bug --- src/models/qwen.js | 131 +++++++++++++++++++++++++++------------------ 1 file changed, 80 insertions(+), 51 deletions(-) diff --git a/src/models/qwen.js b/src/models/qwen.js index d68f579..d4930b5 100644 --- a/src/models/qwen.js +++ b/src/models/qwen.js @@ -1,92 +1,121 @@ +// This code uses Dashscope and HTTP to ensure the latest support for the Qwen model. +// Qwen is also compatible with the OpenAI API format; + import { getKey } from '../utils/keys.js'; export class Qwen { - constructor(model_name, url) { - this.model_name = model_name; + constructor(modelName, url) { + // Initialize model name and API URL + this.modelName = modelName; this.url = url || 'https://dashscope.aliyuncs.com/api/v1/services/aigc/text-generation/generation'; - this.apiKey = getKey('QWEN_API_KEY'); + this.apiKey = getKey('QWEN_API_KEY'); // Get API key from utility function } - async sendRequest(turns, systemMessage, stop_seq = '***') { + async sendRequest(turns, systemMessage, stopSeq = '***', retryCount = 0) { + // Limit retry attempts to avoid infinite recursion + if (retryCount > 5) { + console.error('Maximum retry attempts reached.'); + return 'Error: Too many retry attempts.'; + } + + // Build request data const data = { - model: this.model_name || 'qwen-plus', + model: this.modelName || 'qwen-plus', input: { messages: [{ role: 'system', content: systemMessage }, ...turns] }, - parameters: { result_format: 'message', stop: stop_seq }, + parameters: { result_format: 'message', stop: stopSeq }, }; - const headers = { - 'Authorization': `Bearer ${this.apiKey}`, - 'Content-Type': 'application/json', - }; + // Add default user message if all messages are 'system' role + if (turns.every((msg) => msg.role === 'system')) { + data.input.messages.push({ role: 'user', content: 'hello' }); + } + + // Validate data format before sending request + if (!data.model || !data.input || !data.input.messages || !data.parameters) { + console.error('Invalid request data format:', data); + throw new Error('Invalid request data format.'); + } try { - console.log('Awaiting Qwen API response...'); - const response = await fetch(this.url, { - method: 'POST', - headers, - body: JSON.stringify(data), - }); + // Send request to API + const response = await this._makeHttpRequest(this.url, data); + const choice = response?.output?.choices?.[0]; - if (!response.ok) { - console.error(`Request failed with status ${response.status}: ${response.statusText}`); - return `Request failed with status ${response.status}: ${response.statusText}`; + // Retry request if response is incomplete due to length limit + if (choice?.finish_reason === 'length' && turns.length > 0) { + return this.sendRequest(turns.slice(1), systemMessage, stopSeq, retryCount + 1); } - const responseData = await response.json(); - const choice = responseData?.output?.choices?.[0]; - - if (choice?.finish_reason === 'length') { - console.log('Context length exceeded'); - return this.sendRequest(turns.slice(1), systemMessage, stop_seq); - } - - console.log('Received.'); + // Return response content or default message return choice?.message?.content || 'No content received.'; } catch (err) { + // Error handling, log error and return error message console.error('Error occurred:', err); - return 'My brain disconnected, try again.'; + return 'An error occurred, please try again.'; } } async embed(text) { + // Validate embedding input if (!text || typeof text !== 'string') { - console.error('Invalid input for embedding: text must be a non-empty string.'); - return 'Invalid input for embedding: text must be a non-empty string.'; + console.error('Invalid embedding input: text must be a non-empty string.'); + return 'Invalid embedding input: text must be a non-empty string.'; } + // Build embedding request data const data = { model: 'text-embedding-v2', input: { texts: [text] }, parameters: { text_type: 'query' }, }; + // Validate data format before sending request + if (!data.model || !data.input || !data.input.texts || !data.parameters) { + console.error('Invalid embedding request data format:', data); + throw new Error('Invalid embedding request data format.'); + } + + try { + // Send embedding request to API + const response = await this._makeHttpRequest(this.url, data); + const embedding = response?.output?.embeddings?.[0]?.embedding; + return embedding || 'No embedding result received.'; + } catch (err) { + // Error handling, log error and return error message + console.error('Error occurred:', err); + return 'An error occurred, please try again.'; + } + } + + async _makeHttpRequest(url, data) { + // Set request headers, including authorization and content type const headers = { 'Authorization': `Bearer ${this.apiKey}`, 'Content-Type': 'application/json', }; + // Send HTTP POST request to API + const response = await fetch(url, { + method: 'POST', + headers, + body: JSON.stringify(data), + }); + + // Check response status code, throw error if not successful + if (!response.ok) { + const errorText = await response.text(); + console.error(`Request failed, status code ${response.status}: ${response.statusText}`); + console.error('Error response content:', errorText); + throw new Error(`Request failed, status code ${response.status}: ${response.statusText}`); + } + + // Parse and return response JSON + const responseText = await response.text(); try { - const response = await fetch(this.url, { - method: 'POST', - headers, - body: JSON.stringify(data), - }); - - if (!response.ok) { - console.error(`Request failed with status ${response.status}: ${response.statusText}`); - return `Request failed with status ${response.status}: ${response.statusText}`; - } - - const responseData = await response.json(); - if (!responseData?.output?.embeddings) { - console.error('Invalid response from embedding API'); - return 'An error occurred while processing your embedding request. Please try again.'; - } - - return responseData.output.embeddings[0].embedding; + return JSON.parse(responseText); } catch (err) { - console.error('Error occurred:', err); - return 'An error occurred while processing your embedding request. Please try again.'; + console.error('Failed to parse response JSON:', err); + throw new Error('Invalid response JSON format.'); } } } From 53e6097ecb4a74bbd9f769c2ee24cd0696212b50 Mon Sep 17 00:00:00 2001 From: Qu Yi <2020218037@mail.hfut.edu.cn> Date: Tue, 29 Oct 2024 12:39:10 +0800 Subject: [PATCH 23/59] Update qwen.js Add compatibility tips for openai api. --- src/models/qwen.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/models/qwen.js b/src/models/qwen.js index d4930b5..c343253 100644 --- a/src/models/qwen.js +++ b/src/models/qwen.js @@ -1,5 +1,7 @@ // This code uses Dashscope and HTTP to ensure the latest support for the Qwen model. -// Qwen is also compatible with the OpenAI API format; +// Qwen is also compatible with the OpenAI API format, and the base URL to be configured is: "https://dashscope.aliyuncs.com/compatible-mode/v1". + +https://dashscope.aliyuncs.com/compatible-mode/v1 import { getKey } from '../utils/keys.js'; From 60bc2299eab5f87cb2a5deb09adb52a6e7b1c482 Mon Sep 17 00:00:00 2001 From: bartek Date: Tue, 29 Oct 2024 09:11:02 +0100 Subject: [PATCH 24/59] Create docker-compose.yml added docker-compose.yml --- docker-compose.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7d6243e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,9 @@ +version: '3.8' + +services: + app: + image: node:latest + working_dir: /app + volumes: + - .:/app + command: node main.js From 906df6f6e4585227c5dc136b451f8730aea0c7d0 Mon Sep 17 00:00:00 2001 From: bartek Date: Tue, 29 Oct 2024 09:17:44 +0100 Subject: [PATCH 25/59] Updated README.md: updated docker instruction plus some formatting --- README.md | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 1e9f5cd..484d68f 100644 --- a/README.md +++ b/README.md @@ -50,19 +50,23 @@ You can configure project details in `settings.js`. [See file for more details]( If you intent to `allow_insecure_coding`, it might be a good idea to put the whole app into a docker container to reduce risks of running unknown code. -``` +```bash docker run -i -t --rm -v $(pwd):/app -w /app node:latest node main.js ``` +or simply +```bash +docker-compose up +``` When running in docker, if you want the bot to join your local minecraft server, you have to use a special host address `host.docker.internal` to call your localhost from inside your docker container. Put this into your [settings.js](settings.js): -``` +```javascript "host": "host.docker.internal", // instead of "localhost", to join your local minecraft from inside the docker container ``` ### Online Servers To connect to online servers your bot will need an official Microsoft/Minecraft account. You can use your own personal one, but will need another account if you want to connect with it. Here are example settings for this: -``` +```javascript "host": "111.222.333.444", "port": 55920, "auth": "microsoft", @@ -83,13 +87,15 @@ Bot profiles are json files (such as `andy.json`) that define: By default, the program will use the profiles specified in `settings.js`. You can specify one or more agent profiles using the `--profiles` argument: -`node main.js --profiles ./profiles/andy.json ./profiles/jill.json` +```bash +node main.js --profiles ./profiles/andy.json ./profiles/jill.json +``` ### Model Specifications LLM backends can be specified as simply as `"model": "gpt-3.5-turbo"`. However, for both the chat model and the embedding model, the bot profile can specify the below attributes: -``` +```json "model": { "api": "openai", "url": "https://api.openai.com/v1/", @@ -108,15 +114,15 @@ If the embedding field is not specified, then it will use the default embedding Thus, all the below specifications are equivalent to the above example: -``` +```json "model": "gpt-3.5-turbo" ``` -``` +```json "model": { "api": "openai" } ``` -``` +```json "model": "gpt-3.5-turbo", "embedding": "openai" ``` From 2628f1a4dedc99bbff3fd4c944fa9f018da4bfc6 Mon Sep 17 00:00:00 2001 From: bartek szabat Date: Tue, 29 Oct 2024 11:35:41 +0100 Subject: [PATCH 26/59] run in docker-compose: open prismarine viewer ports --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 7d6243e..0bef712 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,3 +7,5 @@ services: volumes: - .:/app command: node main.js + ports: + - "3000-3003:3000-3003" From 2ee94d7b60a10a0c91541198df7786212ad172e5 Mon Sep 17 00:00:00 2001 From: bartek szabat Date: Tue, 29 Oct 2024 11:40:18 +0100 Subject: [PATCH 27/59] run in docker-compose: open prismarine viewer ports: update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c26b92..414887c 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ You can configure project details in `settings.js`. [See file for more details]( If you intent to `allow_insecure_coding`, it might be a good idea to put the whole app into a docker container to reduce risks of running unknown code. ```bash -docker run -i -t --rm -v $(pwd):/app -w /app node:latest node main.js +docker run -i -t --rm -v $(pwd):/app -w /app -p 3000-3003:3000-3003 node:latest node main.js ``` or simply ```bash From af53f854f8910584e1e8a7db2786cf1f330ab8a0 Mon Sep 17 00:00:00 2001 From: bartek szabat Date: Tue, 29 Oct 2024 11:58:11 +0100 Subject: [PATCH 28/59] run in docker-compose: open prismarine viewer ports: update readme --- README.md | 2 ++ docker-compose.yml | 9 ++++++++- services/viaproxy/README.md | 25 +++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 services/viaproxy/README.md diff --git a/README.md b/README.md index 414887c..018d144 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,8 @@ When running in docker, if you want the bot to join your local minecraft server, "host": "host.docker.internal", // instead of "localhost", to join your local minecraft from inside the docker container ``` +To connect to an unsupported minecraft version, you can try to use [viaproxy](services/viaproxy/README.md) + ### Online Servers To connect to online servers your bot will need an official Microsoft/Minecraft account. You can use your own personal one, but will need another account if you want to connect with it. Here are example settings for this: ```javascript diff --git a/docker-compose.yml b/docker-compose.yml index 0bef712..b37cef5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,5 +7,12 @@ services: volumes: - .:/app command: node main.js + + viaproxy: #use this service to connect to an unsupported minecraft server versions. more info: ./services/viaproxy/README.md + image: ghcr.io/viaversion/viaproxy:latest + volumes: + - ./services/viaproxy:/app/run ports: - - "3000-3003:3000-3003" + - "25568:25568" + profiles: + - viaproxy diff --git a/services/viaproxy/README.md b/services/viaproxy/README.md new file mode 100644 index 0000000..5a9f0cc --- /dev/null +++ b/services/viaproxy/README.md @@ -0,0 +1,25 @@ +Use this service to connect your bot to unsupported minecraft server versions. + +Run: + +```bash +docker-compose --profile viaproxy up +``` + +After first start it will create config file `viaproxy.yml` in this directory. + +Edit this file, and change your desired target `target-address`, + +then point to your `settings.js` `host` and `port` to viaproxy: + +```javascript + "host": "host.docker.internal", + "port": 25568, +``` + +This easily works with "offline" servers. + +Connecting to "online" servers involves more configuration: see `auth-method` in `viaproxy.yml` (TODO describe) + + + From 62550aa2c825534f739c2be0662a7b4d7225248e Mon Sep 17 00:00:00 2001 From: bartek szabat Date: Tue, 29 Oct 2024 11:58:11 +0100 Subject: [PATCH 29/59] run in docker-compose: added viaproxy service to connect to unsupported minecraft versions --- README.md | 2 ++ docker-compose.yml | 9 ++++++++- services/viaproxy/README.md | 25 +++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 services/viaproxy/README.md diff --git a/README.md b/README.md index 414887c..018d144 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,8 @@ When running in docker, if you want the bot to join your local minecraft server, "host": "host.docker.internal", // instead of "localhost", to join your local minecraft from inside the docker container ``` +To connect to an unsupported minecraft version, you can try to use [viaproxy](services/viaproxy/README.md) + ### Online Servers To connect to online servers your bot will need an official Microsoft/Minecraft account. You can use your own personal one, but will need another account if you want to connect with it. Here are example settings for this: ```javascript diff --git a/docker-compose.yml b/docker-compose.yml index 0bef712..b37cef5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,5 +7,12 @@ services: volumes: - .:/app command: node main.js + + viaproxy: #use this service to connect to an unsupported minecraft server versions. more info: ./services/viaproxy/README.md + image: ghcr.io/viaversion/viaproxy:latest + volumes: + - ./services/viaproxy:/app/run ports: - - "3000-3003:3000-3003" + - "25568:25568" + profiles: + - viaproxy diff --git a/services/viaproxy/README.md b/services/viaproxy/README.md new file mode 100644 index 0000000..5a9f0cc --- /dev/null +++ b/services/viaproxy/README.md @@ -0,0 +1,25 @@ +Use this service to connect your bot to unsupported minecraft server versions. + +Run: + +```bash +docker-compose --profile viaproxy up +``` + +After first start it will create config file `viaproxy.yml` in this directory. + +Edit this file, and change your desired target `target-address`, + +then point to your `settings.js` `host` and `port` to viaproxy: + +```javascript + "host": "host.docker.internal", + "port": 25568, +``` + +This easily works with "offline" servers. + +Connecting to "online" servers involves more configuration: see `auth-method` in `viaproxy.yml` (TODO describe) + + + From a58f14e3dea3e841adb19dc34d8cf71e570a3577 Mon Sep 17 00:00:00 2001 From: bartek szabat Date: Tue, 29 Oct 2024 12:04:56 +0100 Subject: [PATCH 30/59] run in docker-compose: viaproxy readme --- services/viaproxy/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/viaproxy/README.md b/services/viaproxy/README.md index 5a9f0cc..ada93c0 100644 --- a/services/viaproxy/README.md +++ b/services/viaproxy/README.md @@ -6,7 +6,7 @@ Run: docker-compose --profile viaproxy up ``` -After first start it will create config file `viaproxy.yml` in this directory. +After first start it will create config file `services/viaproxy/viaproxy.yml`. Edit this file, and change your desired target `target-address`, From 0c162e3ce3a670dffff21d6f08aeec3e2939311e Mon Sep 17 00:00:00 2001 From: bartek szabat Date: Tue, 29 Oct 2024 12:35:05 +0100 Subject: [PATCH 31/59] run in docker-compose: ports lost and found --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index b37cef5..72e3ab9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,8 @@ services: volumes: - .:/app command: node main.js + ports: + - "3000-3003:3000-3003" # see what your bot see: http://localhost:3000/ viaproxy: #use this service to connect to an unsupported minecraft server versions. more info: ./services/viaproxy/README.md image: ghcr.io/viaversion/viaproxy:latest From 04c6fdc13b97e8ace62fbb458390552a654751b0 Mon Sep 17 00:00:00 2001 From: bartek szabat Date: Tue, 29 Oct 2024 12:45:54 +0100 Subject: [PATCH 32/59] run in docker-compose: viaproxy: ignore files, fix readme --- .gitignore | 8 +++++++- services/viaproxy/README.md | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index a2aee7e..6dc4554 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,10 @@ code_records/ scratch.js bots/**/action-code/** bots/**/ -keys.json \ No newline at end of file +keys.json +services/viaproxy/jars/** +services/viaproxy/logs/** +services/viaproxy/plugins/** +services/viaproxy/ViaLoader/** +services/viaproxy/saves.json +services/viaproxy/viaproxy.yml diff --git a/services/viaproxy/README.md b/services/viaproxy/README.md index ada93c0..d6e53ba 100644 --- a/services/viaproxy/README.md +++ b/services/viaproxy/README.md @@ -19,7 +19,7 @@ then point to your `settings.js` `host` and `port` to viaproxy: This easily works with "offline" servers. -Connecting to "online" servers involves more configuration: see `auth-method` in `viaproxy.yml` (TODO describe) +Connecting to "online" servers with viaproxy involves more configuration: see `auth-method` in `services/viaproxy/viaproxy.yml` (TODO describe) From 41282daeb22a30b97da60a5a76fc6324fae355a6 Mon Sep 17 00:00:00 2001 From: Lawtro37 <98205608+Lawtro37@users.noreply.github.com> Date: Tue, 29 Oct 2024 22:35:33 +1000 Subject: [PATCH 33/59] modified "!goToDeath" to use new "goToDeath()" skill this feels more clean and the new skill fixes some bugs note: also added "pickupNearbyItems()" so it picks up its items after death. --- src/agent/commands/actions.js | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index ffc61e4..8c6804f 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -299,13 +299,8 @@ export const actionsList = [ name: "!goToDeath", description: "Go to the location of your bot's last death position and retrieve any lost items.", perform: wrapExecution(async (agent) => { - let death_pos = agent.memory_bank.recallPlace('last death position')[0]; // <- dont even ask why this is an array - if (death_pos !== null) { - await skills.goToPosition(agent.bot, death_pos.x, death_pos.y, death_pos.z, 1); - console.log('going to death'); - } else { - skills.log(agent.bot, "No death location saved."); - } + await skills.goToDeath(agent.bot, agent); + await skills.pickupNearbyItems(agent.bot); }) }, { From e83f5051ce95b35f83b11068569fd5283f00d27b Mon Sep 17 00:00:00 2001 From: Lawtro37 <98205608+Lawtro37@users.noreply.github.com> Date: Tue, 29 Oct 2024 22:38:02 +1000 Subject: [PATCH 34/59] added "goToDeath()" skill and fixed bugs shouldn't fail to path find now and is generally cleaner --- src/agent/library/skills.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index b575ff1..fd53bc3 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -1185,6 +1185,25 @@ export async function goToBed(bot) { return true; } +export async function goToDeath(bot, agent) { + /** + * Navigate to the last death point. + * @param {MinecraftBot} bot, reference to the minecraft bot. + * @param {Agent} agent, reference to the agent. + * @returns {Promise} true if the death point was found, false otherwise. + * @example + * await skills.goToDeath(bot); + **/ + let death_pos = agent.memory_bank.recallPlace('last death position')[0]; // <- dont even ask why this is an array + if (death_pos) { + await goToPosition(bot, death_pos.x, death_pos.y, death_pos.z); + log(bot, `You have reached the death point.`); + return true; + } + log(bot, `Could not find the death point.`); + return false; +} + export async function tillAndSow(bot, x, y, z, seedType=null) { /** * Till the ground at the given position and plant the given seed type. From b8921c3520068cb3a92af112e70b89bee89dbfde Mon Sep 17 00:00:00 2001 From: Lawtro37 <98205608+Lawtro37@users.noreply.github.com> Date: Tue, 29 Oct 2024 22:41:43 +1000 Subject: [PATCH 35/59] cleaned up message sent to AI on death reduced message length by removing confusing and unnecessary info such as what it had in its inventory. Added more direct and less confusable instruction to use the !goToDeath action. --- src/agent/agent.js | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index fbe9f47..cf79733 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -266,15 +266,8 @@ export class Agent { if (death_pos) { death_pos_text = `x: ${death_pos.x.toFixed(2)}, y: ${death_pos.y.toFixed(2)}, z: ${death_pos.x.toFixed(2)}`; } - let inventory = this.bot.inventory.slots; - let death_items = ""; - for (let i=0; i { From 678504767fea5e93e4d381cba6cb3b5f1af71d36 Mon Sep 17 00:00:00 2001 From: bartek szabat Date: Tue, 29 Oct 2024 14:27:22 +0100 Subject: [PATCH 36/59] run in docker-compose: funny comment --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 72e3ab9..d9822f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: - .:/app command: node main.js ports: - - "3000-3003:3000-3003" # see what your bot see: http://localhost:3000/ + - "3000-3003:3000-3003" # see the view from the camera mounted on your bot head: http://localhost:3000/ viaproxy: #use this service to connect to an unsupported minecraft server versions. more info: ./services/viaproxy/README.md image: ghcr.io/viaversion/viaproxy:latest From 7d4d79b035a08036d2469cf4b02fd0d28fbc7497 Mon Sep 17 00:00:00 2001 From: Qu Yi Date: Wed, 30 Oct 2024 15:47:19 +0800 Subject: [PATCH 37/59] Cleaning up the annotations --- src/models/qwen.js | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/src/models/qwen.js b/src/models/qwen.js index c343253..d3d7bec 100644 --- a/src/models/qwen.js +++ b/src/models/qwen.js @@ -1,26 +1,21 @@ // This code uses Dashscope and HTTP to ensure the latest support for the Qwen model. -// Qwen is also compatible with the OpenAI API format, and the base URL to be configured is: "https://dashscope.aliyuncs.com/compatible-mode/v1". - -https://dashscope.aliyuncs.com/compatible-mode/v1 +// Qwen is also compatible with the OpenAI API format; import { getKey } from '../utils/keys.js'; export class Qwen { constructor(modelName, url) { - // Initialize model name and API URL this.modelName = modelName; this.url = url || 'https://dashscope.aliyuncs.com/api/v1/services/aigc/text-generation/generation'; - this.apiKey = getKey('QWEN_API_KEY'); // Get API key from utility function + this.apiKey = getKey('QWEN_API_KEY'); } async sendRequest(turns, systemMessage, stopSeq = '***', retryCount = 0) { - // Limit retry attempts to avoid infinite recursion if (retryCount > 5) { console.error('Maximum retry attempts reached.'); return 'Error: Too many retry attempts.'; } - // Build request data const data = { model: this.modelName || 'qwen-plus', input: { messages: [{ role: 'system', content: systemMessage }, ...turns] }, @@ -32,78 +27,65 @@ export class Qwen { data.input.messages.push({ role: 'user', content: 'hello' }); } - // Validate data format before sending request if (!data.model || !data.input || !data.input.messages || !data.parameters) { console.error('Invalid request data format:', data); throw new Error('Invalid request data format.'); } try { - // Send request to API const response = await this._makeHttpRequest(this.url, data); const choice = response?.output?.choices?.[0]; - // Retry request if response is incomplete due to length limit if (choice?.finish_reason === 'length' && turns.length > 0) { return this.sendRequest(turns.slice(1), systemMessage, stopSeq, retryCount + 1); } - // Return response content or default message return choice?.message?.content || 'No content received.'; } catch (err) { - // Error handling, log error and return error message console.error('Error occurred:', err); return 'An error occurred, please try again.'; } } async embed(text) { - // Validate embedding input if (!text || typeof text !== 'string') { console.error('Invalid embedding input: text must be a non-empty string.'); return 'Invalid embedding input: text must be a non-empty string.'; } - // Build embedding request data const data = { model: 'text-embedding-v2', input: { texts: [text] }, parameters: { text_type: 'query' }, }; - // Validate data format before sending request if (!data.model || !data.input || !data.input.texts || !data.parameters) { console.error('Invalid embedding request data format:', data); throw new Error('Invalid embedding request data format.'); } try { - // Send embedding request to API const response = await this._makeHttpRequest(this.url, data); const embedding = response?.output?.embeddings?.[0]?.embedding; return embedding || 'No embedding result received.'; } catch (err) { - // Error handling, log error and return error message console.error('Error occurred:', err); return 'An error occurred, please try again.'; } } async _makeHttpRequest(url, data) { - // Set request headers, including authorization and content type const headers = { 'Authorization': `Bearer ${this.apiKey}`, 'Content-Type': 'application/json', }; - // Send HTTP POST request to API const response = await fetch(url, { method: 'POST', headers, body: JSON.stringify(data), }); - // Check response status code, throw error if not successful if (!response.ok) { const errorText = await response.text(); console.error(`Request failed, status code ${response.status}: ${response.statusText}`); @@ -111,7 +93,6 @@ export class Qwen { throw new Error(`Request failed, status code ${response.status}: ${response.statusText}`); } - // Parse and return response JSON const responseText = await response.text(); try { return JSON.parse(responseText); From a1a8a38576e8078bd10707d22c6dd0ed3d34cabc Mon Sep 17 00:00:00 2001 From: bartek szabat Date: Wed, 30 Oct 2024 09:31:21 +0100 Subject: [PATCH 38/59] run in docker-compose: readme --- services/viaproxy/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/viaproxy/README.md b/services/viaproxy/README.md index d6e53ba..d1ed687 100644 --- a/services/viaproxy/README.md +++ b/services/viaproxy/README.md @@ -1,4 +1,4 @@ -Use this service to connect your bot to unsupported minecraft server versions. +Use this service to connect your bot to an unsupported minecraft server versions. Run: @@ -10,7 +10,7 @@ After first start it will create config file `services/viaproxy/viaproxy.yml`. Edit this file, and change your desired target `target-address`, -then point to your `settings.js` `host` and `port` to viaproxy: +then point your `settings.js` `host` and `port` to viaproxy endpoint: ```javascript "host": "host.docker.internal", @@ -19,7 +19,7 @@ then point to your `settings.js` `host` and `port` to viaproxy: This easily works with "offline" servers. -Connecting to "online" servers with viaproxy involves more configuration: see `auth-method` in `services/viaproxy/viaproxy.yml` (TODO describe) +Connecting to "online" servers via viaproxy involves more effort: see `auth-method` in `services/viaproxy/viaproxy.yml` (TODO describe) From f827bb0a208c18dc412e12f31639454a32af72fe Mon Sep 17 00:00:00 2001 From: Lawtro37 <98205608+Lawtro37@users.noreply.github.com> Date: Wed, 30 Oct 2024 22:18:17 +1000 Subject: [PATCH 39/59] tell the AI what dimension it died in hopefully this should help it not try to find its stuff in the nether in the otherworld. --- src/agent/agent.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index cf79733..745c08f 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -266,8 +266,9 @@ export class Agent { if (death_pos) { death_pos_text = `x: ${death_pos.x.toFixed(2)}, y: ${death_pos.y.toFixed(2)}, z: ${death_pos.x.toFixed(2)}`; } + let dimention = this.bot.game.dimension; this.history.add('system', `You died at position ${death_pos_text || "unknown"} with the final message: '${message}'. Previous actions were stopped and you have since respawned.`); - this.handleMessage('system', `You died at position ${death_pos_text || "unknown"} with the final message: '${message}'. Previous actions were stopped and you have respawned. Notify the user and perform any necessary actions. use !goToDeath to return to death position. (this will automaticly take you to your death position and pick up your items)`); + this.handleMessage('system', `You died at position ${death_pos_text || "unknown"} in the dimention ${dimention} with the final message: '${message}'. Previous actions were stopped and you have respawned. Notify the user and perform any necessary actions. use !goToDeath to return to death position. (this will automaticly take you to your death position and pick up your items)`); } }); this.bot.on('idle', () => { From 015bf686ee82bb8b4a7286d3f35da71c9124c717 Mon Sep 17 00:00:00 2001 From: Lawtro37 <98205608+Lawtro37@users.noreply.github.com> Date: Wed, 30 Oct 2024 23:01:46 +1000 Subject: [PATCH 40/59] fixed bug were pathfinding wouldn't work fixed race condition with "await" gets death position before attempting to path find to death position --- src/agent/library/skills.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index fd53bc3..e379f10 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -1194,7 +1194,7 @@ export async function goToDeath(bot, agent) { * @example * await skills.goToDeath(bot); **/ - let death_pos = agent.memory_bank.recallPlace('last death position')[0]; // <- dont even ask why this is an array + let death_pos = await agent.memory_bank.recallPlace('last death position')[0]; // <- dont even ask why this is an array if (death_pos) { await goToPosition(bot, death_pos.x, death_pos.y, death_pos.z); log(bot, `You have reached the death point.`); From fa505e422a1356162e1492d112b88ccfbc4fa840 Mon Sep 17 00:00:00 2001 From: JurassikLizard Date: Wed, 30 Oct 2024 13:33:25 -0400 Subject: [PATCH 41/59] Added recipe item hints --- src/agent/library/skills.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index b575ff1..2b8d233 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -43,6 +43,11 @@ export async function craftRecipe(bot, itemName, num=1) { **/ let placedTable = false; + if (mc.getItemCraftingRecipes(itemName).length == 0) { + log(bot, `${itemName} is either not an item, or it does not have a crafting recipe!`); + return false; + } + // get recipes that don't require a crafting table let recipes = bot.recipesFor(mc.getItemId(itemName), null, 1, null); let craftingTable = null; @@ -76,7 +81,7 @@ export async function craftRecipe(bot, itemName, num=1) { } } if (!recipes || recipes.length === 0) { - log(bot, `You do not have the resources to craft a ${itemName}.`); + log(bot, `You do not have the resources to craft a ${itemName}. It requires: ${Object.entries(mc.getItemCraftingRecipes(itemName)[0]).map(([key, value]) => `${key}: ${value}`).join(', ')}.`); if (placedTable) { await collectBlock(bot, 'crafting_table', 1); } From deaec43771a99a9da20466f1d0d99877213dbc22 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Sat, 2 Nov 2024 11:01:43 -0500 Subject: [PATCH 42/59] dont translate when in english --- src/utils/translator.js | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/utils/translator.js b/src/utils/translator.js index 879cfe4..bc9cc77 100644 --- a/src/utils/translator.js +++ b/src/utils/translator.js @@ -1,18 +1,14 @@ import translate from 'google-translate-api-x'; import settings from '../../settings.js'; -const preferred_lang = settings.language; +const preferred_lang = String(settings.language).toLowerCase(); export async function handleTranslation(message) { + if (preferred_lang === 'en' || preferred_lang === 'english') + return message; try { - if (preferred_lang.toLowerCase() === 'en' || preferred_lang.toLowerCase() === 'english') { - return message; - } else { - const lang = String(preferred_lang); - - const translation = await translate(message, { to: lang }); - return translation.text || message; - } + const translation = await translate(message, { to: preferred_lang }); + return translation.text || message; } catch (error) { console.error('Error translating message:', error); return message; @@ -20,6 +16,8 @@ export async function handleTranslation(message) { } export async function handleEnglishTranslation(message) { + if (preferred_lang === 'en' || preferred_lang === 'english') + return message; try { const translation = await translate(message, { to: 'english' }); return translation.text || message; From 012f229936912c850b8535482722d56df66524d2 Mon Sep 17 00:00:00 2001 From: Max Robinson Date: Sat, 2 Nov 2024 13:03:58 -0500 Subject: [PATCH 43/59] Better instructions and order --- README.md | 73 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 56a1360..01b6df5 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,42 @@ # Mindcraft 🧠⛏️ -Crafting minds for Minecraft with Language Models and Mineflayer! +Crafting minds for Minecraft with LLMs and Mineflayer! [Join the discord for support!](https://discord.gg/ZsrAAByEnr) #### ‼️Warning‼️ -This project allows an AI model to write/execute code on your computer that may be insecure, dangerous, and vulnerable to injection attacks on public servers. Code writing is disabled by default, you can enable it by setting `allow_insecure_coding` to `true` in `settings.js`. Enable only on local or private servers, **never** on public servers. Ye be warned. +This project allows an LLM to write/execute code on your computer. While the code is sandboxed, it may still be insecure and vulnerable to injection attacks on public servers. Code writing is disabled by default, you can enable it by setting `allow_insecure_coding` to `true` in `settings.js`. If you connect to a public server with coding enabled, we strongly recommend running with additional layers of security such as docker containers. Ye be warned. ## Requirements -- [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) (up to v1.21.1) -- [Node.js](https://nodejs.org/) (at least v14) -- One of these: [OpenAI API Key](https://openai.com/blog/openai-api) | [Gemini API Key](https://aistudio.google.com/app/apikey) |[Anthropic API Key](https://docs.anthropic.com/claude/docs/getting-access-to-claude) | [Replicate API Key](https://replicate.com/) | [Hugging Face API Key](https://huggingface.co/) | [Groq API Key](https://console.groq.com/keys) | [Ollama Installed](https://ollama.com/download). |[Qwen API Key [Intl.]](https://www.alibabacloud.com/help/en/model-studio/developer-reference/get-api-key)/[[cn]](https://help.aliyun.com/zh/model-studio/getting-started/first-api-call-to-qwen?) | +- [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) (up to v1.21.1, recommend v1.20.4) +- [Node.js Installed](https://nodejs.org/) (at least v14) +- One of these: [OpenAI API Key](https://openai.com/blog/openai-api) | [Gemini API Key](https://aistudio.google.com/app/apikey) | [Anthropic API Key](https://docs.anthropic.com/claude/docs/getting-access-to-claude) | [Replicate API Key](https://replicate.com/) | [Hugging Face API Key](https://huggingface.co/) | [Groq API Key](https://console.groq.com/keys) | [Ollama Installed](https://ollama.com/download). | [Qwen API Key [Intl.]](https://www.alibabacloud.com/help/en/model-studio/developer-reference/get-api-key)/[[cn]](https://help.aliyun.com/zh/model-studio/getting-started/first-api-call-to-qwen?) | -## Installation +## Install and Run + +1. Make sure you have the requirements above. + +2. Clone or download this repository (big green button) + +3. Rename `keys.example.json` to `keys.json` and fill in your API keys (you only need one). The desired model is set in `andy.json` or other profiles. For other models refer to the table below. + +4. In terminal/command prompt, run `npm install` from the installed directory + +5. Start a minecraft world and open it to LAN on localhost port `55916` + +6. Run `node main.js` from the installed directory + +## Customization + +You can configure project details in `settings.js`. [See file.](settings.js) + +You can configure the agent's name, model, and prompts in their profile like `andy.json`. -Rename `keys.example.json` to `keys.json` and fill in your API keys, and you can set the desired model in `andy.json` or other profiles. | API | Config Variable | Example Model name | Docs | |------|------|------|------| -| OpenAI | `OPENAI_API_KEY` | `gpt-3.5-turbo` | [docs](https://platform.openai.com/docs/models) | +| OpenAI | `OPENAI_API_KEY` | `gpt-4o-mini` | [docs](https://platform.openai.com/docs/models) | | Google | `GEMINI_API_KEY` | `gemini-pro` | [docs](https://ai.google.dev/gemini-api/docs/models/gemini) | | Anthropic | `ANTHROPIC_API_KEY` | `claude-3-haiku-20240307` | [docs](https://docs.anthropic.com/claude/docs/models-overview) | | Replicate | `REPLICATE_API_KEY` | `meta/meta-llama-3-70b-instruct` | [docs](https://replicate.com/collections/language-models) | @@ -31,25 +48,20 @@ Rename `keys.example.json` to `keys.json` and fill in your API keys, and you can If you use Ollama, to install the models used by default (generation and embedding), execute the following terminal command: `ollama pull llama3 && ollama pull nomic-embed-text` -Then, clone/download this repository +## Online Servers +To connect to online servers your bot will need an official Microsoft/Minecraft account. You can use your own personal one, but will need another account if you want to connect too and play with it. To connect, change these lines in `settings.js`: +```javascript +"host": "111.222.333.444", +"port": 55920, +"auth": "microsoft", -Run `npm install` from the installed directory +// rest is same... +``` +‼️ The bot's name in the profile.json must exactly match the Minecraft profile name! Otherwise the bot will spam talk to itself. -Install the minecraft version specified in `settings.js`, currently supports up to 1.21.1 +### Docker Container -### Running Locally - -Start a minecraft world and open it to LAN on localhost port `55916` - -Run `node main.js` - -You can configure the agent's name, model, and prompts in their profile like `andy.json`. - -You can configure project details in `settings.js`. [See file for more details](settings.js) - -### Run in docker to reduce some of the risks - -If you intent to `allow_insecure_coding`, it might be a good idea to put the whole app into a docker container to reduce risks of running unknown code. +If you intend to `allow_insecure_coding`, it is a good idea to run the app in a docker container to reduce risks of running unknown code. This is strongly recommended before connecting to remote servers. ```bash docker run -i -t --rm -v $(pwd):/app -w /app -p 3000-3003:3000-3003 node:latest node main.js @@ -67,18 +79,7 @@ When running in docker, if you want the bot to join your local minecraft server, To connect to an unsupported minecraft version, you can try to use [viaproxy](services/viaproxy/README.md) -### Online Servers -To connect to online servers your bot will need an official Microsoft/Minecraft account. You can use your own personal one, but will need another account if you want to connect with it. Here are example settings for this: -```javascript -"host": "111.222.333.444", -"port": 55920, -"auth": "microsoft", - -// rest is same... -``` -‼️ Please make sure your bot's name in the profile.json matches the account name! Otherwise the bot will spam talk to itself. - -### Bot Profiles +## Bot Profiles Bot profiles are json files (such as `andy.json`) that define: From 771f5de70896d068f93c3fe09659df056ec39523 Mon Sep 17 00:00:00 2001 From: Max Robinson Date: Sat, 2 Nov 2024 13:37:41 -0500 Subject: [PATCH 44/59] Create FAQ.md --- FAQ.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 FAQ.md diff --git a/FAQ.md b/FAQ.md new file mode 100644 index 0000000..380e44b --- /dev/null +++ b/FAQ.md @@ -0,0 +1,18 @@ +# Common Issues +- `Error: connect ECONNREFUSED`: Minecraft refused to connect with mindcraft program. Most likely due to: + - you have not opened your game to LAN in game settings + - you have the wrong version of minecraft, make sure your MC version is the same as specified in `settings.js` + +- `ERR_MODULE_NOT_FOUND`: You are missing an npm package. run `npm install` + +- `My brain disconnected, try again`: Something is wrong with the LLM api. You may have the wrong API key, exceeded your rate limits, or other. Check the program outputs for more details. + +- `I'm stuck!` or other issues with constantly getting stuck: + - Mineflayer's pathfinder is imperfect. We have improved upon it with patches, but these might not have been applied properly. Make sure your code is up to date with main, delete the `node_modules` folder, and run `npm install` + - The bot will still get stuck occasionally, but not constantly. +# Common Questions +- Mod Support? Mindcraft only supports client-side mods like optifine and sodium, though they can be tricky to set up. Mods that change minecraft game mechanics are not supported. + +- Texture Packs? Apparently these cause issues and refuse to connect. Not sure why + +- Baritone? Baritone is a mod that is completely different from mineflayer. There is currently no easy way to integrate the two programs. From 7fd505c4b597314076c681aa7c4f012ad6ab6d02 Mon Sep 17 00:00:00 2001 From: Max Robinson Date: Sat, 2 Nov 2024 14:08:59 -0500 Subject: [PATCH 45/59] Added faq link --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 01b6df5..b37d430 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,10 @@ Crafting minds for Minecraft with LLMs and Mineflayer! -[Join the discord for support!](https://discord.gg/ZsrAAByEnr) +[FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) + +[Discord Support](https://discord.gg/ZsrAAByEnr) + #### ‼️Warning‼️ @@ -28,6 +31,8 @@ This project allows an LLM to write/execute code on your computer. While the cod 6. Run `node main.js` from the installed directory +If you encounter issues, check the [FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) or find support on [discord](https://discord.gg/ZsrAAByEnr). We are currently not very responsive to github issues. + ## Customization You can configure project details in `settings.js`. [See file.](settings.js) @@ -59,6 +64,8 @@ To connect to online servers your bot will need an official Microsoft/Minecraft ``` ‼️ The bot's name in the profile.json must exactly match the Minecraft profile name! Otherwise the bot will spam talk to itself. +To use different accounts, Mindcraft will connect with the account that the Minecraft launcher is currently using. You can switch accounts in the launcer, then run `node main.js`, then switch to your main account after the bot has connected. + ### Docker Container If you intend to `allow_insecure_coding`, it is a good idea to run the app in a docker container to reduce risks of running unknown code. This is strongly recommended before connecting to remote servers. From 32706b00f7041788aaa5b3fa65794b6a30b51740 Mon Sep 17 00:00:00 2001 From: Max Robinson Date: Sat, 2 Nov 2024 14:10:37 -0500 Subject: [PATCH 46/59] Update FAQ.md --- FAQ.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/FAQ.md b/FAQ.md index 380e44b..903cd25 100644 --- a/FAQ.md +++ b/FAQ.md @@ -1,15 +1,19 @@ # Common Issues - `Error: connect ECONNREFUSED`: Minecraft refused to connect with mindcraft program. Most likely due to: - you have not opened your game to LAN in game settings + - your LAN port is incorrect, make sure the one you enter in game is the same as specified in `settings.js` - you have the wrong version of minecraft, make sure your MC version is the same as specified in `settings.js` - `ERR_MODULE_NOT_FOUND`: You are missing an npm package. run `npm install` +- Many issues are caused by out-of-date node module patches, especially after updates. A catch-all is to delete the `node_modules` folder, and run `npm install` + - `My brain disconnected, try again`: Something is wrong with the LLM api. You may have the wrong API key, exceeded your rate limits, or other. Check the program outputs for more details. - `I'm stuck!` or other issues with constantly getting stuck: - Mineflayer's pathfinder is imperfect. We have improved upon it with patches, but these might not have been applied properly. Make sure your code is up to date with main, delete the `node_modules` folder, and run `npm install` - The bot will still get stuck occasionally, but not constantly. + # Common Questions - Mod Support? Mindcraft only supports client-side mods like optifine and sodium, though they can be tricky to set up. Mods that change minecraft game mechanics are not supported. From 3e6e73f6a181b5b0b40f5f908d976120b6843e28 Mon Sep 17 00:00:00 2001 From: youngdev <26363920+Shandelier@users.noreply.github.com> Date: Sat, 2 Nov 2024 22:24:23 +0100 Subject: [PATCH 47/59] gemini gains IQ (embedding fix) --- src/models/gemini.js | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/models/gemini.js b/src/models/gemini.js index ee4dfa4..1536d66 100644 --- a/src/models/gemini.js +++ b/src/models/gemini.js @@ -6,6 +6,28 @@ export class Gemini { constructor(model_name, url) { this.model_name = model_name; this.url = url; + this.safetySettings = [ + { + "category": "HARM_CATEGORY_DANGEROUS", + "threshold": "BLOCK_NONE", + }, + { + "category": "HARM_CATEGORY_HARASSMENT", + "threshold": "BLOCK_NONE", + }, + { + "category": "HARM_CATEGORY_HATE_SPEECH", + "threshold": "BLOCK_NONE", + }, + { + "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "threshold": "BLOCK_NONE", + }, + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_NONE", + }, + ]; this.genAI = new GoogleGenerativeAI(getKey('GEMINI_API_KEY')); } @@ -14,12 +36,14 @@ export class Gemini { let model; if (this.url) { model = this.genAI.getGenerativeModel( - {model: this.model_name || "gemini-pro"}, - {baseUrl: this.url} + { model: this.model_name || "gemini-1.5-flash" }, + { baseUrl: this.url }, + { safetySettings: this.safetySettings } ); } else { model = this.genAI.getGenerativeModel( - {model: this.model_name || "gemini-pro"} + { model: this.model_name || "gemini-1.5-flash" }, + { safetySettings: this.safetySettings } ); } @@ -39,16 +63,16 @@ export class Gemini { let model; if (this.url) { model = this.genAI.getGenerativeModel( - {model: this.model_name || "embedding-001"}, - {baseUrl: this.url} + { model: "text-embedding-004" }, + { baseUrl: this.url } ); } else { model = this.genAI.getGenerativeModel( - {model: this.model_name || "embedding-001"} + { model: "text-embedding-004" } ); } const result = await model.embedContent(text); - return result.embedding; + return result.embedding.values; } } \ No newline at end of file From b6856e63c8b836ea5a935e8dbbb8f9e910522cd8 Mon Sep 17 00:00:00 2001 From: Max Robinson Date: Sat, 2 Nov 2024 17:37:17 -0500 Subject: [PATCH 48/59] reinforced coding warning --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b37d430..7e4c047 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Crafting minds for Minecraft with LLMs and Mineflayer! #### ‼️Warning‼️ -This project allows an LLM to write/execute code on your computer. While the code is sandboxed, it may still be insecure and vulnerable to injection attacks on public servers. Code writing is disabled by default, you can enable it by setting `allow_insecure_coding` to `true` in `settings.js`. If you connect to a public server with coding enabled, we strongly recommend running with additional layers of security such as docker containers. Ye be warned. +Do not connect this bot to public servers with coding enabled. This project allows an LLM to write/execute code on your computer. While the code is sandboxed, it is still vulnerable to injection attacks on public servers. Code writing is disabled by default, you can enable it by setting `allow_insecure_coding` to `true` in `settings.js`. We strongly recommend running with additional layers of security such as docker containers. Ye be warned. ## Requirements From eb0719d4e378ca72bfb04cb6d5d8a8f7e782389b Mon Sep 17 00:00:00 2001 From: Lawtro37 <98205608+Lawtro37@users.noreply.github.com> Date: Sun, 3 Nov 2024 12:28:20 +1000 Subject: [PATCH 49/59] modified message sent to AI modified message sent to AI fixed miss spelling of "dimension" removed code to save to memory as this is already done by "handleMessage()" --- src/agent/agent.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index 745c08f..60f2406 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -261,14 +261,13 @@ export class Agent { if (jsonMsg.translate && jsonMsg.translate.startsWith('death') && message.startsWith(this.name)) { console.log('Agent died: ', message); let death_pos = this.bot.entity.position; - this.memory_bank.rememberPlace('last death position', death_pos); + this.memory_bank.rememberPlace('last_death_position', death_pos); let death_pos_text = null; if (death_pos) { death_pos_text = `x: ${death_pos.x.toFixed(2)}, y: ${death_pos.y.toFixed(2)}, z: ${death_pos.x.toFixed(2)}`; } let dimention = this.bot.game.dimension; - this.history.add('system', `You died at position ${death_pos_text || "unknown"} with the final message: '${message}'. Previous actions were stopped and you have since respawned.`); - this.handleMessage('system', `You died at position ${death_pos_text || "unknown"} in the dimention ${dimention} with the final message: '${message}'. Previous actions were stopped and you have respawned. Notify the user and perform any necessary actions. use !goToDeath to return to death position. (this will automaticly take you to your death position and pick up your items)`); + this.handleMessage('system', `You died at position ${death_pos_text || "unknown"} in the ${dimention} dimension with the final message: '${message}'. Your place of death is saved as 'last_death_position' if you want to return. Previous actions were stopped and you have respawned.`); } }); this.bot.on('idle', () => { From ac545acf5115dd75698bcf61fde03902df724207 Mon Sep 17 00:00:00 2001 From: Lawtro37 <98205608+Lawtro37@users.noreply.github.com> Date: Sun, 3 Nov 2024 17:38:48 +1000 Subject: [PATCH 50/59] remove unnecessary "goToDeath()" skill this code is unnecessary the bot can use "!goToPlace('last_death_position')" --- src/agent/library/skills.js | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index e379f10..b575ff1 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -1185,25 +1185,6 @@ export async function goToBed(bot) { return true; } -export async function goToDeath(bot, agent) { - /** - * Navigate to the last death point. - * @param {MinecraftBot} bot, reference to the minecraft bot. - * @param {Agent} agent, reference to the agent. - * @returns {Promise} true if the death point was found, false otherwise. - * @example - * await skills.goToDeath(bot); - **/ - let death_pos = await agent.memory_bank.recallPlace('last death position')[0]; // <- dont even ask why this is an array - if (death_pos) { - await goToPosition(bot, death_pos.x, death_pos.y, death_pos.z); - log(bot, `You have reached the death point.`); - return true; - } - log(bot, `Could not find the death point.`); - return false; -} - export async function tillAndSow(bot, x, y, z, seedType=null) { /** * Till the ground at the given position and plant the given seed type. From 5606dad576c0781c75b5f6eff238b12dec541e63 Mon Sep 17 00:00:00 2001 From: Lawtro37 <98205608+Lawtro37@users.noreply.github.com> Date: Sun, 3 Nov 2024 17:40:04 +1000 Subject: [PATCH 51/59] removed unnecessary "!goToDeath" action this code is unnecessary the bot can use "!goToPlace('last_death_position')" --- src/agent/commands/actions.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index 8c6804f..a80ad29 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -295,14 +295,6 @@ export const actionsList = [ await skills.goToBed(agent.bot); }) }, - { - name: "!goToDeath", - description: "Go to the location of your bot's last death position and retrieve any lost items.", - perform: wrapExecution(async (agent) => { - await skills.goToDeath(agent.bot, agent); - await skills.pickupNearbyItems(agent.bot); - }) - }, { name: '!activate', description: 'Activate the nearest object of a given type.', From d12f604ef5ca905aa1b388f85092a9b751dcd9f0 Mon Sep 17 00:00:00 2001 From: Qu Yi Date: Mon, 4 Nov 2024 00:06:32 +0800 Subject: [PATCH 52/59] Update FAQ.md Add Common Issues: Why I added the api key but still prompted that the key can't be found? --- FAQ.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/FAQ.md b/FAQ.md index 903cd25..d560b03 100644 --- a/FAQ.md +++ b/FAQ.md @@ -13,6 +13,11 @@ - `I'm stuck!` or other issues with constantly getting stuck: - Mineflayer's pathfinder is imperfect. We have improved upon it with patches, but these might not have been applied properly. Make sure your code is up to date with main, delete the `node_modules` folder, and run `npm install` - The bot will still get stuck occasionally, but not constantly. + +- `Why I added the api key but still prompted that the key can't be found?` + - Possible reason 1: Did not modify keys.example.json to keys.json. + - Possible reason 2: If you use vscode to edit, you need to `ctrl+s` to save the file for the changes to take effect. + - Possible reason 3: Not setting the code path correctly in setting.js, use andy.js by default. # Common Questions - Mod Support? Mindcraft only supports client-side mods like optifine and sodium, though they can be tricky to set up. Mods that change minecraft game mechanics are not supported. From 25a0f5c0696b1210e51777128192f5dbfd965ee0 Mon Sep 17 00:00:00 2001 From: JurassikLizard Date: Sun, 3 Nov 2024 12:03:12 -0500 Subject: [PATCH 53/59] rename task -> action --- src/agent/{tasks.js => actions.js} | 56 ++++++++++++++--------------- src/agent/coder.js | 4 +-- src/agent/commands/actions.js | 57 +++++++++++++++--------------- src/agent/modes.js | 4 +-- src/agent/npc/build_goal.js | 2 +- src/agent/npc/controller.js | 10 +++--- src/agent/self_prompter.js | 2 +- 7 files changed, 67 insertions(+), 68 deletions(-) rename src/agent/{tasks.js => actions.js} (72%) diff --git a/src/agent/tasks.js b/src/agent/actions.js similarity index 72% rename from src/agent/tasks.js rename to src/agent/actions.js index ab02bd3..ca25aee 100644 --- a/src/agent/tasks.js +++ b/src/agent/actions.js @@ -1,23 +1,23 @@ -export class TaskManager { +export class ActionManager { constructor(agent) { this.agent = agent; this.executing = false; - this.currentTaskLabel = ''; - this.currentTaskFn = null; + this.currentActionLabel = ''; + this.currentActionFn = null; this.timedout = false; this.resume_func = null; this.resume_name = ''; } - async resumeTask(taskFn, timeout) { - return this._executeResume(taskFn, timeout); + async resumeAction(actionFn, timeout) { + return this._executeResume(actionFn, timeout); } - async runTask(taskLabel, taskFn, { timeout, resume = false } = {}) { + async runAction(actionLabel, actionFn, { timeout, resume = false } = {}) { if (resume) { - return this._executeResume(taskFn, timeout); + return this._executeResume(actionFn, timeout); } else { - return this._executeTask(taskLabel, taskFn, timeout); + return this._executeAction(actionLabel, actionFn, timeout); } } @@ -40,31 +40,31 @@ export class TaskManager { this.resume_name = null; } - async _executeResume(taskFn = null, timeout = 10) { - const new_resume = taskFn != null; + async _executeResume(actionFn = null, timeout = 10) { + const new_resume = actionFn != null; if (new_resume) { // start new resume - this.resume_func = taskFn; - this.resume_name = this.currentTaskLabel; + this.resume_func = actionFn; + this.resume_name = this.currentActionLabel; } if (this.resume_func != null && this.agent.isIdle() && (!this.agent.self_prompter.on || new_resume)) { - this.currentTaskLabel = this.resume_name; - let res = await this._executeTask(this.resume_name, this.resume_func, timeout); - this.currentTaskLabel = ''; + this.currentActionLabel = this.resume_name; + let res = await this._executeAction(this.resume_name, this.resume_func, timeout); + this.currentActionLabel = ''; return res; } else { return { success: false, message: null, interrupted: false, timedout: false }; } } - async _executeTask(taskLabel, taskFn, timeout = 10) { + async _executeAction(actionLabel, actionFn, timeout = 10) { let TIMEOUT; try { console.log('executing code...\n'); - // await current task to finish (executing=false), with 10 seconds timeout + // await current action to finish (executing=false), with 10 seconds timeout // also tell agent.bot to stop various actions if (this.executing) { - console.log(`new task "${taskLabel}" trying to interrupt current task "${this.currentTaskLabel}"`); + console.log(`new action "${actionLabel}" trying to interrupt current action "${this.currentActionLabel}"`); } await this.stop(); @@ -72,21 +72,21 @@ export class TaskManager { this.agent.clearBotLogs(); this.executing = true; - this.currentTaskLabel = taskLabel; - this.currentTaskFn = taskFn; + this.currentActionLabel = actionLabel; + this.currentActionFn = actionFn; // timeout in minutes if (timeout > 0) { TIMEOUT = this._startTimeout(timeout); } - // start the task - await taskFn(); + // start the action + await actionFn(); - // mark task as finished + cleanup + // mark action as finished + cleanup this.executing = false; - this.currentTaskLabel = ''; - this.currentTaskFn = null; + this.currentActionLabel = ''; + this.currentActionFn = null; clearTimeout(TIMEOUT); // get bot activity summary @@ -100,12 +100,12 @@ export class TaskManager { this.agent.bot.emit('idle'); } - // return task status report + // return action status report return { success: true, message: output, interrupted, timedout }; } catch (err) { this.executing = false; - this.currentTaskLabel = ''; - this.currentTaskFn = null; + this.currentActionLabel = ''; + this.currentActionFn = null; clearTimeout(TIMEOUT); this.cancelResume(); console.error("Code execution triggered catch: " + err); diff --git a/src/agent/coder.js b/src/agent/coder.js index 9c91107..3413bad 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -94,7 +94,7 @@ export class Coder { async generateCode(agent_history) { // wrapper to prevent overlapping code generation loops - await this.agent.tasks.stop(); + await this.agent.actions.stop(); this.generating = true; let res = await this.generateCodeLoop(agent_history); this.generating = false; @@ -154,7 +154,7 @@ export class Coder { return {success: false, message: null, interrupted: false, timedout: false}; } - code_return = await this.agent.tasks.runTask('newAction', async () => { + code_return = await this.agent.actions.runAction('newAction', async () => { return await executionModuleExports.main(this.agent.bot); }, { timeout: settings.code_timeout_mins }); if (code_return.interrupted && !code_return.timedout) diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index a9bc46c..4e69b35 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -1,12 +1,12 @@ import * as skills from '../library/skills.js'; import settings from '../../../settings.js'; -function runAsTask (taskLabel, taskFn, resume = false, timeout = -1) { +function runAsAction (taskLabel, actionFn, resume = false, timeout = -1) { return async function (agent, ...args) { - const taskFnWithAgent = async () => { - await taskFn(agent, ...args); + const actionFnWithAgent = async () => { + await actionFn(agent, ...args); }; - const code_return = await agent.tasks.runTask(`action:${taskLabel}`, taskFnWithAgent, { timeout, resume }); + const code_return = await agent.actions.runAction(`action:${taskLabel}`, actionFnWithAgent, { timeout, resume }); if (code_return.interrupted && !code_return.timedout) return; return code_return.message; @@ -31,9 +31,9 @@ export const actionsList = [ name: '!stop', description: 'Force stop all actions and commands that are currently executing.', perform: async function (agent) { - await agent.tasks.stop(); + await agent.actions.stop(); agent.clearBotLogs(); - agent.tasks.cancelResume(); + agent.actions.cancelResume(); agent.bot.emit('idle'); let msg = 'Agent stopped.'; if (agent.self_prompter.on) @@ -73,7 +73,7 @@ export const actionsList = [ 'player_name': {type: 'string', description: 'The name of the player to go to.'}, 'closeness': {type: 'float', description: 'How close to get to the player.', domain: [0, Infinity]} }, - perform: runAsTask('goToPlayer', async (agent, player_name, closeness) => { + perform: runAsAction('goToPlayer', async (agent, player_name, closeness) => { return await skills.goToPlayer(agent.bot, player_name, closeness); }) }, @@ -84,7 +84,7 @@ export const actionsList = [ 'player_name': {type: 'string', description: 'name of the player to follow.'}, 'follow_dist': {type: 'float', description: 'The distance to follow from.', domain: [0, Infinity]} }, - perform: runAsTask('followPlayer', async (agent, player_name, follow_dist) => { + perform: runAsAction('followPlayer', async (agent, player_name, follow_dist) => { await skills.followPlayer(agent.bot, player_name, follow_dist); }, true) }, @@ -96,7 +96,7 @@ export const actionsList = [ 'closeness': { type: 'float', description: 'How close to get to the block.', domain: [0, Infinity] }, 'search_range': { type: 'float', description: 'The distance to search for the block.', domain: [0, Infinity] } }, - perform: runAsTask('goToBlock', async (agent, type, closeness, range) => { + perform: runAsAction('goToBlock', async (agent, type, closeness, range) => { await skills.goToNearestBlock(agent.bot, type, closeness, range); }) }, @@ -104,7 +104,7 @@ export const actionsList = [ name: '!moveAway', description: 'Move away from the current location in any direction by a given distance.', params: {'distance': { type: 'float', description: 'The distance to move away.', domain: [0, Infinity] }}, - perform: runAsTask('moveAway', async (agent, distance) => { + perform: runAsAction('moveAway', async (agent, distance) => { await skills.moveAway(agent.bot, distance); }) }, @@ -122,7 +122,7 @@ export const actionsList = [ name: '!goToPlace', description: 'Go to a saved location.', params: {'name': { type: 'string', description: 'The name of the location to go to.' }}, - perform: runAsTask('goToPlace', async (agent, name) => { + perform: runAsAction('goToPlace', async (agent, name) => { const pos = agent.memory_bank.recallPlace(name); if (!pos) { skills.log(agent.bot, `No location named "${name}" saved.`); @@ -139,7 +139,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the item to give.' }, 'num': { type: 'int', description: 'The number of items to give.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsTask('givePlayer', async (agent, player_name, item_name, num) => { + perform: runAsAction('givePlayer', async (agent, player_name, item_name, num) => { await skills.giveToPlayer(agent.bot, item_name, player_name, num); }) }, @@ -147,7 +147,7 @@ export const actionsList = [ name: '!consume', description: 'Eat/drink the given item.', params: {'item_name': { type: 'ItemName', description: 'The name of the item to consume.' }}, - perform: runAsTask('consume', async (agent, item_name) => { + perform: runAsAction('consume', async (agent, item_name) => { await agent.bot.consume(item_name); skills.log(agent.bot, `Consumed ${item_name}.`); }) @@ -156,7 +156,7 @@ export const actionsList = [ name: '!equip', description: 'Equip the given item.', params: {'item_name': { type: 'ItemName', description: 'The name of the item to equip.' }}, - perform: runAsTask('equip', async (agent, item_name) => { + perform: runAsAction('equip', async (agent, item_name) => { await skills.equip(agent.bot, item_name); }) }, @@ -167,7 +167,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the item to put in the chest.' }, 'num': { type: 'int', description: 'The number of items to put in the chest.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsTask('putInChest', async (agent, item_name, num) => { + perform: runAsAction('putInChest', async (agent, item_name, num) => { await skills.putInChest(agent.bot, item_name, num); }) }, @@ -178,7 +178,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the item to take.' }, 'num': { type: 'int', description: 'The number of items to take.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsTask('takeFromChest', async (agent, item_name, num) => { + perform: runAsAction('takeFromChest', async (agent, item_name, num) => { await skills.takeFromChest(agent.bot, item_name, num); }) }, @@ -186,7 +186,7 @@ export const actionsList = [ name: '!viewChest', description: 'View the items/counts of the nearest chest.', params: { }, - perform: runAsTask('viewChest', async (agent) => { + perform: runAsAction('viewChest', async (agent) => { await skills.viewChest(agent.bot); }) }, @@ -197,7 +197,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the item to discard.' }, 'num': { type: 'int', description: 'The number of items to discard.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsTask('discard', async (agent, item_name, num) => { + perform: runAsAction('discard', async (agent, item_name, num) => { const start_loc = agent.bot.entity.position; await skills.moveAway(agent.bot, 5); await skills.discard(agent.bot, item_name, num); @@ -211,7 +211,7 @@ export const actionsList = [ 'type': { type: 'BlockName', description: 'The block type to collect.' }, 'num': { type: 'int', description: 'The number of blocks to collect.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsTask('collectBlocks', async (agent, type, num) => { + perform: runAsAction('collectBlocks', async (agent, type, num) => { await skills.collectBlock(agent.bot, type, num); }, false, 10) // 10 minute timeout }, @@ -221,7 +221,7 @@ export const actionsList = [ params: { 'type': { type: 'BlockName', description: 'The block type to collect.' } }, - perform: runAsTask('collectAllBlocks', async (agent, type) => { + perform: runAsAction('collectAllBlocks', async (agent, type) => { let success = await skills.collectBlock(agent.bot, type, 1); if (!success) agent.tasks.cancelResume(); @@ -234,7 +234,7 @@ export const actionsList = [ 'recipe_name': { type: 'ItemName', description: 'The name of the output item to craft.' }, 'num': { type: 'int', description: 'The number of times to craft the recipe. This is NOT the number of output items, as it may craft many more items depending on the recipe.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsTask('craftRecipe', async (agent, recipe_name, num) => { + perform: runAsAction('craftRecipe', async (agent, recipe_name, num) => { await skills.craftRecipe(agent.bot, recipe_name, num); }) }, @@ -245,7 +245,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the input item to smelt.' }, 'num': { type: 'int', description: 'The number of times to smelt the item.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsTask('smeltItem', async (agent, item_name, num) => { + perform: runAsAction('smeltItem', async (agent, item_name, num) => { let response = await skills.smeltItem(agent.bot, item_name, num); if (response.indexOf('Successfully') !== -1) { // there is a bug where the bot's inventory is not updated after smelting @@ -258,9 +258,8 @@ export const actionsList = [ { name: '!clearFurnace', description: 'Take all items out of the nearest furnace.', - description: 'Take all items out of the nearest furnace.', params: { }, - perform: runAsTask('clearFurnace', async (agent) => { + perform: runAsAction('clearFurnace', async (agent) => { await skills.clearNearestFurnace(agent.bot); }) }, @@ -268,7 +267,7 @@ export const actionsList = [ name: '!placeHere', description: 'Place a given block in the current location. Do NOT use to build structures, only use for single blocks/torches.', params: {'type': { type: 'BlockName', description: 'The block type to place.' }}, - perform: runAsTask('placeHere', async (agent, type) => { + perform: runAsAction('placeHere', async (agent, type) => { let pos = agent.bot.entity.position; await skills.placeBlock(agent.bot, type, pos.x, pos.y, pos.z); }) @@ -277,14 +276,14 @@ export const actionsList = [ name: '!attack', description: 'Attack and kill the nearest entity of a given type.', params: {'type': { type: 'string', description: 'The type of entity to attack.'}}, - perform: runAsTask('attack', async (agent, type) => { + perform: runAsAction('attack', async (agent, type) => { await skills.attackNearest(agent.bot, type, true); }) }, { name: '!goToBed', description: 'Go to the nearest bed and sleep.', - perform: runAsTask('goToBed', async (agent) => { + perform: runAsAction('goToBed', async (agent) => { await skills.goToBed(agent.bot); }) }, @@ -292,7 +291,7 @@ export const actionsList = [ name: '!activate', description: 'Activate the nearest object of a given type.', params: {'type': { type: 'BlockName', description: 'The type of object to activate.' }}, - perform: runAsTask('activate', async (agent, type) => { + perform: runAsAction('activate', async (agent, type) => { await skills.activateNearestBlock(agent.bot, type); }) }, @@ -300,7 +299,7 @@ export const actionsList = [ name: '!stay', description: 'Stay in the current location no matter what. Pauses all modes.', params: {'type': { type: 'int', description: 'The number of seconds to stay. -1 for forever.', domain: [-1, Number.MAX_SAFE_INTEGER] }}, - perform: runAsTask('stay', async (agent, seconds) => { + perform: runAsAction('stay', async (agent, seconds) => { await skills.stay(agent.bot, seconds); }) }, diff --git a/src/agent/modes.js b/src/agent/modes.js index 977d01d..f3005c7 100644 --- a/src/agent/modes.js +++ b/src/agent/modes.js @@ -260,7 +260,7 @@ async function execute(mode, agent, func, timeout=-1) { if (agent.self_prompter.on) agent.self_prompter.stopLoop(); mode.active = true; - let code_return = await agent.tasks.runTask(`mode:${mode.name}`, async () => { + let code_return = await agent.actions.runAction(`mode:${mode.name}`, async () => { await func(); }, { timeout }); mode.active = false; @@ -328,7 +328,7 @@ class ModeController { this.unPauseAll(); } for (let mode of this.modes_list) { - let interruptible = mode.interrupts.some(i => i === 'all') || mode.interrupts.some(i => `action:${i}` === this.agent.tasks.currentTaskLabel); + let interruptible = mode.interrupts.some(i => i === 'all') || mode.interrupts.some(i => `action:${i}` === this.agent.actions.currentActionLabel); if (mode.on && !mode.paused && !mode.active && (this.agent.isIdle() || interruptible)) { await mode.update(this.agent); } diff --git a/src/agent/npc/build_goal.js b/src/agent/npc/build_goal.js index e7199ee..ebca78f 100644 --- a/src/agent/npc/build_goal.js +++ b/src/agent/npc/build_goal.js @@ -13,7 +13,7 @@ export class BuildGoal { async wrapSkill(func) { if (!this.agent.isIdle()) return false; - let res = await this.agent.tasks.runTask('BuildGoal', func); + let res = await this.agent.actions.runAction('BuildGoal', func); return !res.interrupted; } diff --git a/src/agent/npc/controller.js b/src/agent/npc/controller.js index d7f213c..227328e 100644 --- a/src/agent/npc/controller.js +++ b/src/agent/npc/controller.js @@ -72,7 +72,7 @@ export class NPCContoller { if (!this.agent.isIdle()) return; // Persue goal - if (!this.agent.tasks.resume_func) { + if (!this.agent.actions.resume_func) { this.executeNext(); this.agent.history.save(); } @@ -104,7 +104,7 @@ export class NPCContoller { async executeNext() { if (!this.agent.isIdle()) return; - await this.agent.tasks.runTask('npc:moveAway', async () => { + await this.agent.actions.runAction('npc:moveAway', async () => { await skills.moveAway(this.agent.bot, 2); }); @@ -114,7 +114,7 @@ export class NPCContoller { if (building == this.data.home) { let door_pos = this.getBuildingDoor(building); if (door_pos) { - await this.agent.tasks.runTask('npc:exitBuilding', async () => { + await this.agent.actions.runAction('npc:exitBuilding', async () => { await skills.useDoor(this.agent.bot, door_pos); await skills.moveAway(this.agent.bot, 2); // If the bot is too close to the building it will try to enter again }); @@ -132,13 +132,13 @@ export class NPCContoller { let building = this.currentBuilding(); if (this.data.home !== null && (building === null || building != this.data.home)) { let door_pos = this.getBuildingDoor(this.data.home); - await this.agent.tasks.runTask('npc:returnHome', async () => { + await this.agent.actions.runAction('npc:returnHome', async () => { await skills.useDoor(this.agent.bot, door_pos); }); } // Go to bed - await this.agent.tasks.runTask('npc:bed', async () => { + await this.agent.actions.runAction('npc:bed', async () => { await skills.goToBed(this.agent.bot); }); } diff --git a/src/agent/self_prompter.js b/src/agent/self_prompter.js index 4f9d885..8c928e1 100644 --- a/src/agent/self_prompter.js +++ b/src/agent/self_prompter.js @@ -87,7 +87,7 @@ export class SelfPrompter { async stop(stop_action=true) { this.interrupt = true; if (stop_action) - await this.agent.tasks.stop(); + await this.agent.actions.stop(); await this.stopLoop(); this.on = false; } From e13fd4afc4b5d97d63603c44bd9f631355ddf4a3 Mon Sep 17 00:00:00 2001 From: JurassikLizard Date: Sun, 3 Nov 2024 12:03:56 -0500 Subject: [PATCH 54/59] extra refactor cleanup --- src/agent/commands/actions.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index 4e69b35..3a2e453 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -1,12 +1,12 @@ import * as skills from '../library/skills.js'; import settings from '../../../settings.js'; -function runAsAction (taskLabel, actionFn, resume = false, timeout = -1) { +function runAsAction (actionLabel, actionFn, resume = false, timeout = -1) { return async function (agent, ...args) { const actionFnWithAgent = async () => { await actionFn(agent, ...args); }; - const code_return = await agent.actions.runAction(`action:${taskLabel}`, actionFnWithAgent, { timeout, resume }); + const code_return = await agent.actions.runAction(`action:${actionLabel}`, actionFnWithAgent, { timeout, resume }); if (code_return.interrupted && !code_return.timedout) return; return code_return.message; @@ -224,7 +224,7 @@ export const actionsList = [ perform: runAsAction('collectAllBlocks', async (agent, type) => { let success = await skills.collectBlock(agent.bot, type, 1); if (!success) - agent.tasks.cancelResume(); + agent.actions.cancelResume(); }, true, 3) // 3 minute timeout }, { From 434f295a9e2f26fcb0bdeacad841952e00f9bf6f Mon Sep 17 00:00:00 2001 From: JurassikLizard Date: Sun, 3 Nov 2024 12:04:47 -0500 Subject: [PATCH 55/59] missing agent.js refactors --- src/agent/agent.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index 30715d8..99f7f00 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -4,7 +4,7 @@ import { Prompter } from './prompter.js'; import { initModes } from './modes.js'; import { initBot } from '../utils/mcdata.js'; import { containsCommand, commandExists, executeCommand, truncCommandMessage, isAction } from './commands/index.js'; -import { TaskManager } from './tasks.js'; +import { ActionManager } from './actions.js'; import { NPCContoller } from './npc/controller.js'; import { MemoryBank } from './memory_bank.js'; import { SelfPrompter } from './self_prompter.js'; @@ -14,7 +14,7 @@ import settings from '../../settings.js'; export class Agent { async start(profile_fp, load_mem=false, init_message=null, count_id=0) { - this.tasks = new TaskManager(this); + this.actions = new ActionManager(this); this.prompter = new Prompter(this, profile_fp); this.name = this.prompter.getName(); this.history = new History(this); @@ -263,8 +263,8 @@ export class Agent { this.cleanKill('Bot disconnected! Killing agent process.'); }); this.bot.on('death', () => { - this.tasks.cancelResume(); - this.tasks.stop(); + this.actions.cancelResume(); + this.actions.stop(); }); this.bot.on('kicked', (reason) => { console.warn('Bot kicked!', reason); @@ -280,7 +280,7 @@ export class Agent { this.bot.clearControlStates(); this.bot.pathfinder.stop(); // clear any lingering pathfinder this.bot.modes.unPauseAll(); - this.tasks.resumeTask(); + this.actions.resumeAction(); }); // Init NPC controller @@ -310,7 +310,7 @@ export class Agent { } isIdle() { - return !this.tasks.executing && !this.coder.generating; + return !this.actions.executing && !this.coder.generating; } cleanKill(msg='Killing agent process...') { From 6a17de7bf67c2cf68b46e9180c84081c77f95695 Mon Sep 17 00:00:00 2001 From: JurassikLizard Date: Sun, 3 Nov 2024 12:08:15 -0500 Subject: [PATCH 56/59] forgotten item_goal.js --- src/agent/npc/item_goal.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent/npc/item_goal.js b/src/agent/npc/item_goal.js index 9d67e7d..40589ba 100644 --- a/src/agent/npc/item_goal.js +++ b/src/agent/npc/item_goal.js @@ -322,7 +322,7 @@ export class ItemGoal { // If the bot has failed to obtain the block before, explore if (this.failed.includes(next.name)) { this.failed = this.failed.filter((item) => item !== next.name); - await this.agent.tasks.runTask('itemGoal:explore', async () => { + await this.agent.actions.runAction('itemGoal:explore', async () => { await skills.moveAway(this.agent.bot, 8); }); } else { @@ -339,7 +339,7 @@ export class ItemGoal { // Execute the next goal let init_quantity = world.getInventoryCounts(this.agent.bot)[next.name] || 0; - await this.agent.tasks.runTask('itemGoal:next', async () => { + await this.agent.actions.runAction('itemGoal:next', async () => { await next.execute(quantity); }); let final_quantity = world.getInventoryCounts(this.agent.bot)[next.name] || 0; From 19f8238aa1def5247257e169c796db775dbb2ee2 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Sun, 3 Nov 2024 22:17:28 -0600 Subject: [PATCH 57/59] renamed file, refactor labeling --- src/agent/{actions.js => action_manager.js} | 24 ++++---- src/agent/agent.js | 4 +- src/agent/commands/actions.js | 62 ++++++++++++--------- src/agent/modes.js | 6 +- 4 files changed, 53 insertions(+), 43 deletions(-) rename src/agent/{actions.js => action_manager.js} (87%) diff --git a/src/agent/actions.js b/src/agent/action_manager.js similarity index 87% rename from src/agent/actions.js rename to src/agent/action_manager.js index ca25aee..833f3c0 100644 --- a/src/agent/actions.js +++ b/src/agent/action_manager.js @@ -15,7 +15,7 @@ export class ActionManager { async runAction(actionLabel, actionFn, { timeout, resume = false } = {}) { if (resume) { - return this._executeResume(actionFn, timeout); + return this._executeResume(actionLabel, actionFn, timeout); } else { return this._executeAction(actionLabel, actionFn, timeout); } @@ -23,28 +23,28 @@ export class ActionManager { async stop() { if (!this.executing) return; - console.trace(); - const start = Date.now(); + const timeout = setTimeout(() => { + this.agent.cleanKill('Code execution refused stop after 10 seconds. Killing process.'); + }, 10000); while (this.executing) { - this.agent.interruptBot(); + this.agent.requestInterrupt(); console.log('waiting for code to finish executing...'); - await new Promise(resolve => setTimeout(resolve, 1000)); - if (Date.now() - start > 10 * 1000) { - this.agent.cleanKill('Code execution refused stop after 10 seconds. Killing process.'); - } + await new Promise(resolve => setTimeout(resolve, 300)); } - } + clearTimeout(timeout); + } cancelResume() { this.resume_func = null; this.resume_name = null; } - async _executeResume(actionFn = null, timeout = 10) { + async _executeResume(actionLabel = null, actionFn = null, timeout = 10) { const new_resume = actionFn != null; if (new_resume) { // start new resume this.resume_func = actionFn; - this.resume_name = this.currentActionLabel; + assert(actionLabel != null, 'actionLabel is required for new resume'); + this.resume_name = actionLabel; } if (this.resume_func != null && this.agent.isIdle() && (!this.agent.self_prompter.on || new_resume)) { this.currentActionLabel = this.resume_name; @@ -64,7 +64,7 @@ export class ActionManager { // await current action to finish (executing=false), with 10 seconds timeout // also tell agent.bot to stop various actions if (this.executing) { - console.log(`new action "${actionLabel}" trying to interrupt current action "${this.currentActionLabel}"`); + console.log(`action "${actionLabel}" trying to interrupt current action "${this.currentActionLabel}"`); } await this.stop(); diff --git a/src/agent/agent.js b/src/agent/agent.js index 99f7f00..d821ecb 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -4,7 +4,7 @@ import { Prompter } from './prompter.js'; import { initModes } from './modes.js'; import { initBot } from '../utils/mcdata.js'; import { containsCommand, commandExists, executeCommand, truncCommandMessage, isAction } from './commands/index.js'; -import { ActionManager } from './actions.js'; +import { ActionManager } from './action_manager.js'; import { NPCContoller } from './npc/controller.js'; import { MemoryBank } from './memory_bank.js'; import { SelfPrompter } from './self_prompter.js'; @@ -93,7 +93,7 @@ export class Agent { }); } - interruptBot() { + requestInterrupt() { this.bot.interrupt_code = true; this.bot.collectBlock.cancelTask(); this.bot.pathfinder.stop(); diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index 3a2e453..ab34894 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -1,8 +1,16 @@ import * as skills from '../library/skills.js'; import settings from '../../../settings.js'; -function runAsAction (actionLabel, actionFn, resume = false, timeout = -1) { - return async function (agent, ...args) { +function runAsAction (actionFn, resume = false, timeout = -1) { + let actionLabel = null; // Will be set on first use + + const wrappedAction = async function (agent, ...args) { + // Set actionLabel only once, when the action is first created + if (!actionLabel) { + const actionObj = actionsList.find(a => a.perform === wrappedAction); + actionLabel = actionObj.name.substring(1); // Remove the ! prefix + } + const actionFnWithAgent = async () => { await actionFn(agent, ...args); }; @@ -11,6 +19,8 @@ function runAsAction (actionLabel, actionFn, resume = false, timeout = -1) { return; return code_return.message; } + + return wrappedAction; } export const actionsList = [ @@ -73,18 +83,18 @@ export const actionsList = [ 'player_name': {type: 'string', description: 'The name of the player to go to.'}, 'closeness': {type: 'float', description: 'How close to get to the player.', domain: [0, Infinity]} }, - perform: runAsAction('goToPlayer', async (agent, player_name, closeness) => { + perform: runAsAction(async (agent, player_name, closeness) => { return await skills.goToPlayer(agent.bot, player_name, closeness); }) }, { name: '!followPlayer', - description: 'Endlessly follow the given player. Will defend that player if self_defense mode is on.', + description: 'Endlessly follow the given player.', params: { 'player_name': {type: 'string', description: 'name of the player to follow.'}, 'follow_dist': {type: 'float', description: 'The distance to follow from.', domain: [0, Infinity]} }, - perform: runAsAction('followPlayer', async (agent, player_name, follow_dist) => { + perform: runAsAction(async (agent, player_name, follow_dist) => { await skills.followPlayer(agent.bot, player_name, follow_dist); }, true) }, @@ -94,9 +104,9 @@ export const actionsList = [ params: { 'type': { type: 'BlockName', description: 'The block type to go to.' }, 'closeness': { type: 'float', description: 'How close to get to the block.', domain: [0, Infinity] }, - 'search_range': { type: 'float', description: 'The distance to search for the block.', domain: [0, Infinity] } + 'search_range': { type: 'float', description: 'The range to search for the block.', domain: [0, 512] } }, - perform: runAsAction('goToBlock', async (agent, type, closeness, range) => { + perform: runAsAction(async (agent, type, closeness, range) => { await skills.goToNearestBlock(agent.bot, type, closeness, range); }) }, @@ -104,7 +114,7 @@ export const actionsList = [ name: '!moveAway', description: 'Move away from the current location in any direction by a given distance.', params: {'distance': { type: 'float', description: 'The distance to move away.', domain: [0, Infinity] }}, - perform: runAsAction('moveAway', async (agent, distance) => { + perform: runAsAction(async (agent, distance) => { await skills.moveAway(agent.bot, distance); }) }, @@ -122,7 +132,7 @@ export const actionsList = [ name: '!goToPlace', description: 'Go to a saved location.', params: {'name': { type: 'string', description: 'The name of the location to go to.' }}, - perform: runAsAction('goToPlace', async (agent, name) => { + perform: runAsAction(async (agent, name) => { const pos = agent.memory_bank.recallPlace(name); if (!pos) { skills.log(agent.bot, `No location named "${name}" saved.`); @@ -139,7 +149,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the item to give.' }, 'num': { type: 'int', description: 'The number of items to give.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsAction('givePlayer', async (agent, player_name, item_name, num) => { + perform: runAsAction(async (agent, player_name, item_name, num) => { await skills.giveToPlayer(agent.bot, item_name, player_name, num); }) }, @@ -147,7 +157,7 @@ export const actionsList = [ name: '!consume', description: 'Eat/drink the given item.', params: {'item_name': { type: 'ItemName', description: 'The name of the item to consume.' }}, - perform: runAsAction('consume', async (agent, item_name) => { + perform: runAsAction(async (agent, item_name) => { await agent.bot.consume(item_name); skills.log(agent.bot, `Consumed ${item_name}.`); }) @@ -156,7 +166,7 @@ export const actionsList = [ name: '!equip', description: 'Equip the given item.', params: {'item_name': { type: 'ItemName', description: 'The name of the item to equip.' }}, - perform: runAsAction('equip', async (agent, item_name) => { + perform: runAsAction(async (agent, item_name) => { await skills.equip(agent.bot, item_name); }) }, @@ -167,7 +177,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the item to put in the chest.' }, 'num': { type: 'int', description: 'The number of items to put in the chest.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsAction('putInChest', async (agent, item_name, num) => { + perform: runAsAction(async (agent, item_name, num) => { await skills.putInChest(agent.bot, item_name, num); }) }, @@ -178,7 +188,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the item to take.' }, 'num': { type: 'int', description: 'The number of items to take.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsAction('takeFromChest', async (agent, item_name, num) => { + perform: runAsAction(async (agent, item_name, num) => { await skills.takeFromChest(agent.bot, item_name, num); }) }, @@ -186,7 +196,7 @@ export const actionsList = [ name: '!viewChest', description: 'View the items/counts of the nearest chest.', params: { }, - perform: runAsAction('viewChest', async (agent) => { + perform: runAsAction(async (agent) => { await skills.viewChest(agent.bot); }) }, @@ -197,7 +207,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the item to discard.' }, 'num': { type: 'int', description: 'The number of items to discard.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsAction('discard', async (agent, item_name, num) => { + perform: runAsAction(async (agent, item_name, num) => { const start_loc = agent.bot.entity.position; await skills.moveAway(agent.bot, 5); await skills.discard(agent.bot, item_name, num); @@ -211,7 +221,7 @@ export const actionsList = [ 'type': { type: 'BlockName', description: 'The block type to collect.' }, 'num': { type: 'int', description: 'The number of blocks to collect.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsAction('collectBlocks', async (agent, type, num) => { + perform: runAsAction(async (agent, type, num) => { await skills.collectBlock(agent.bot, type, num); }, false, 10) // 10 minute timeout }, @@ -221,7 +231,7 @@ export const actionsList = [ params: { 'type': { type: 'BlockName', description: 'The block type to collect.' } }, - perform: runAsAction('collectAllBlocks', async (agent, type) => { + perform: runAsAction(async (agent, type) => { let success = await skills.collectBlock(agent.bot, type, 1); if (!success) agent.actions.cancelResume(); @@ -234,7 +244,7 @@ export const actionsList = [ 'recipe_name': { type: 'ItemName', description: 'The name of the output item to craft.' }, 'num': { type: 'int', description: 'The number of times to craft the recipe. This is NOT the number of output items, as it may craft many more items depending on the recipe.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsAction('craftRecipe', async (agent, recipe_name, num) => { + perform: runAsAction(async (agent, recipe_name, num) => { await skills.craftRecipe(agent.bot, recipe_name, num); }) }, @@ -245,7 +255,7 @@ export const actionsList = [ 'item_name': { type: 'ItemName', description: 'The name of the input item to smelt.' }, 'num': { type: 'int', description: 'The number of times to smelt the item.', domain: [1, Number.MAX_SAFE_INTEGER] } }, - perform: runAsAction('smeltItem', async (agent, item_name, num) => { + perform: runAsAction(async (agent, item_name, num) => { let response = await skills.smeltItem(agent.bot, item_name, num); if (response.indexOf('Successfully') !== -1) { // there is a bug where the bot's inventory is not updated after smelting @@ -259,7 +269,7 @@ export const actionsList = [ name: '!clearFurnace', description: 'Take all items out of the nearest furnace.', params: { }, - perform: runAsAction('clearFurnace', async (agent) => { + perform: runAsAction(async (agent) => { await skills.clearNearestFurnace(agent.bot); }) }, @@ -267,7 +277,7 @@ export const actionsList = [ name: '!placeHere', description: 'Place a given block in the current location. Do NOT use to build structures, only use for single blocks/torches.', params: {'type': { type: 'BlockName', description: 'The block type to place.' }}, - perform: runAsAction('placeHere', async (agent, type) => { + perform: runAsAction(async (agent, type) => { let pos = agent.bot.entity.position; await skills.placeBlock(agent.bot, type, pos.x, pos.y, pos.z); }) @@ -276,14 +286,14 @@ export const actionsList = [ name: '!attack', description: 'Attack and kill the nearest entity of a given type.', params: {'type': { type: 'string', description: 'The type of entity to attack.'}}, - perform: runAsAction('attack', async (agent, type) => { + perform: runAsAction(async (agent, type) => { await skills.attackNearest(agent.bot, type, true); }) }, { name: '!goToBed', description: 'Go to the nearest bed and sleep.', - perform: runAsAction('goToBed', async (agent) => { + perform: runAsAction(async (agent) => { await skills.goToBed(agent.bot); }) }, @@ -291,7 +301,7 @@ export const actionsList = [ name: '!activate', description: 'Activate the nearest object of a given type.', params: {'type': { type: 'BlockName', description: 'The type of object to activate.' }}, - perform: runAsAction('activate', async (agent, type) => { + perform: runAsAction(async (agent, type) => { await skills.activateNearestBlock(agent.bot, type); }) }, @@ -299,7 +309,7 @@ export const actionsList = [ name: '!stay', description: 'Stay in the current location no matter what. Pauses all modes.', params: {'type': { type: 'int', description: 'The number of seconds to stay. -1 for forever.', domain: [-1, Number.MAX_SAFE_INTEGER] }}, - perform: runAsAction('stay', async (agent, seconds) => { + perform: runAsAction(async (agent, seconds) => { await skills.stay(agent.bot, seconds); }) }, diff --git a/src/agent/modes.js b/src/agent/modes.js index f3005c7..f86603a 100644 --- a/src/agent/modes.js +++ b/src/agent/modes.js @@ -162,7 +162,7 @@ const modes = [ { name: 'item_collecting', description: 'Collect nearby items when idle.', - interrupts: ['followPlayer'], + interrupts: ['action:followPlayer'], on: true, active: false, @@ -193,7 +193,7 @@ const modes = [ { name: 'torch_placing', description: 'Place torches when idle and there are no torches nearby.', - interrupts: ['followPlayer'], + interrupts: ['action:followPlayer'], on: true, active: false, cooldown: 5, @@ -328,7 +328,7 @@ class ModeController { this.unPauseAll(); } for (let mode of this.modes_list) { - let interruptible = mode.interrupts.some(i => i === 'all') || mode.interrupts.some(i => `action:${i}` === this.agent.actions.currentActionLabel); + let interruptible = mode.interrupts.some(i => i === 'all') || mode.interrupts.some(i => i === this.agent.actions.currentActionLabel); if (mode.on && !mode.paused && !mode.active && (this.agent.isIdle() || interruptible)) { await mode.update(this.agent); } From 511d87b3bccec6755c191330f18d46f3e4d984cc Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Sun, 3 Nov 2024 22:20:51 -0600 Subject: [PATCH 58/59] removed checksafe again? --- src/agent/coder.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/agent/coder.js b/src/agent/coder.js index 3413bad..f4b7219 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -141,13 +141,6 @@ export class Coder { } code = res.substring(res.indexOf('```')+3, res.lastIndexOf('```')); - if (!checkSafe(code)) { - console.warn(`Detected insecure generated code, not executing. Insecure code: \n\`${code}\``); - const message = 'Error: Code insecurity detected. Do not import, read/write files, execute dynamic code, or access the internet. Please try again:'; - messages.push({ role: 'system', content: message }); - continue; - } - const executionModuleExports = await this.stageCode(code); if (!executionModuleExports) { agent_history.add('system', 'Failed to stage code, something is wrong.'); From f2bb0d9140d07f17fed0b852ed95a1dbb0a4202f Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Sun, 3 Nov 2024 23:01:34 -0600 Subject: [PATCH 59/59] actually save location --- src/agent/agent.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index 60f2406..d4350c9 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -261,7 +261,7 @@ export class Agent { if (jsonMsg.translate && jsonMsg.translate.startsWith('death') && message.startsWith(this.name)) { console.log('Agent died: ', message); let death_pos = this.bot.entity.position; - this.memory_bank.rememberPlace('last_death_position', death_pos); + this.memory_bank.rememberPlace('last_death_position', death_pos.x, death_pos.y, death_pos.z); let death_pos_text = null; if (death_pos) { death_pos_text = `x: ${death_pos.x.toFixed(2)}, y: ${death_pos.y.toFixed(2)}, z: ${death_pos.x.toFixed(2)}`;