From e2e199a093dd66a11f89f1ae92243c137022f6e5 Mon Sep 17 00:00:00 2001 From: Kolby Nottingham Date: Fri, 26 Jan 2024 12:11:32 -0800 Subject: [PATCH 1/7] init defaults --- src/agent/agent.js | 4 ++ src/agent/coder.js | 21 +++++- src/agent/commands/actions.js | 29 +++++++-- src/agent/library/skills.js | 33 +--------- src/agent/modes.js | 118 ++++++++++++++++------------------ 5 files changed, 104 insertions(+), 101 deletions(-) diff --git a/src/agent/agent.js b/src/agent/agent.js index 2b486a3..4ec5805 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -165,6 +165,10 @@ export class Agent { } }); + this.bot.on('idle', () => { + this.coder.executeDefault(); + }); + this.self_defense = true; this.defending = false; this._pause_defending = false; diff --git a/src/agent/coder.js b/src/agent/coder.js index a5d19d4..ec98da0 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -12,6 +12,9 @@ export class Coder { this.executing = false; this.code_template = ''; this.timedout = false; + this.default_func = null; + this.default_name = null; + this.interruptible = false; } async load() { @@ -85,7 +88,6 @@ export class Coder { }); } - async generateCode(agent_history) { let system_message = "You are a minecraft mineflayer bot that plays minecraft by writing javascript codeblocks. Given the conversation between you and the user, use the provided skills and world functions to write your code in a codeblock. Example response: ``` // your code here ``` You will then be given a response to your code. If you are satisfied with the response, respond without a codeblock in a conversational way. If something went wrong, write another codeblock and try to fix the problem."; system_message += getSkillDocs(); @@ -150,6 +152,21 @@ export class Coder { return; } + async executeDefault(func=null, name=null, timeout=10) { + if (func != null) { + this.default_func = func; + this.default_name = name; + } + if (this.default_func != null) { + this.interruptible = true; + let res = await this.execute(this.default_func, timeout); + this.interruptible = false; + return res; + } else { + return {success: false, message: null, interrupted: false, timedout: false}; + } + } + // returns {success: bool, message: string, interrupted: bool, timedout: false} async execute(func, timeout=10) { if (!this.code_template) return {success: false, message: "Code template not loaded.", interrupted: false, timedout: false}; @@ -171,6 +188,7 @@ export class Coder { let interrupted = this.agent.bot.interrupt_code; let timedout = this.timedout; this.clear(); + if (!interrupted) this.agent.bot.emit('idle'); return {success:true, message: output, interrupted, timedout}; } catch (err) { console.error("Code execution triggered catch: " + err); @@ -180,6 +198,7 @@ export class Coder { let message = this.formatOutput(this.agent.bot) + '!!Code threw exception!! Error: ' + err; let interrupted = this.agent.bot.interrupt_code; this.clear(); + if (!interrupted) this.agent.bot.emit('idle'); return {success: false, message, interrupted, timedout: false}; } } diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index 2bb9cb8..49b1ac6 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -1,11 +1,18 @@ import * as skills from '../library/skills.js'; -function wrapExecution(func, timeout=-1) { +function wrapExecution(func, timeout=-1, default_name=null) { return async function (agent, ...args) { - let code_return = await agent.coder.execute(async () => { - await func(agent, ...args); - }, timeout); + let code_return; + if (default_name != null) { + code_return = await agent.coder.executeDefault(async () => { + await func(agent, ...args); + }, default_name, timeout); + } else { + code_return = await agent.coder.execute(async () => { + await func(agent, ...args); + }, timeout); + } if (code_return.interrupted && !code_return.timedout) return; return code_return.message; @@ -26,6 +33,8 @@ export const actionsList = [ perform: async function (agent) { await agent.coder.stop(); agent.coder.clear(); + agent.coder.default_func = null; + agent.coder.default_name = null; return 'Agent stopped.'; } }, @@ -60,7 +69,7 @@ export const actionsList = [ params: {'player_name': '(string) The name of the player to follow.'}, perform: wrapExecution(async (agent, player_name) => { await skills.followPlayer(agent.bot, player_name); - }) + }, -1, 'followPlayer') }, { name: '!givePlayer', @@ -81,6 +90,16 @@ export const actionsList = [ await skills.collectBlock(agent.bot, type, num); }, 10) // 10 minute timeout }, + { + name: '!collectAllBlocks', + description: 'Collect all the nearest blocks of a given type until told to stop.', + params: { + 'type': '(string) The block type to collect. Ex: !collectAllBlocks("stone")' + }, + perform: wrapExecution(async (agent, type) => { + await skills.collectBlock(agent.bot, type, 1); + }, 10, 'collectAllBlocks') // 10 minute timeout + }, { name: '!craftRecipe', description: 'Craft the given recipe a given number of times. Ex: I will craft 8 sticks !craftRecipe("stick", 2)', diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index 8cea083..9753ed3 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -643,49 +643,18 @@ export async function followPlayer(bot, username) { * @example * await skills.followPlayer(bot, "player"); **/ - bot.modes.pause('self_defense'); - bot.modes.pause('hunting'); - let player = bot.players[username].entity if (!player) return false; const follow_distance = 4; - const attack_distance = 8; - bot.pathfinder.setMovements(new pf.Movements(bot)); bot.pathfinder.setGoal(new pf.goals.GoalFollow(player, follow_distance), true); log(bot, `You are now actively following player ${username}.`); while (!bot.interrupt_code) { - let acted = false; - if (bot.modes.isOn('self_defense')) { - const enemy = world.getNearestEntityWhere(bot, entity => mc.isHostile(entity), attack_distance); - if (enemy) { - log(bot, `Found ${enemy.name}, attacking!`, true); - await defendSelf(bot, 8); - acted = true; - } - } - if (bot.modes.isOn('hunting')) { - const animal = world.getNearestEntityWhere(bot, entity => mc.isHuntable(entity), attack_distance); - if (animal) { - log(bot, `Hunting ${animal.name}!`, true); - await attackEntity(bot, animal, true); - acted = true; - } - } - if (bot.entity.position.distanceTo(player.position) < follow_distance) { - acted = autoLight(bot); - } - - if (acted) { // if we did something then resume following - bot.pathfinder.setMovements(new pf.Movements(bot)); - bot.pathfinder.setGoal(new pf.goals.GoalFollow(player, follow_distance), true); - } - await new Promise(resolve => setTimeout(resolve, 500)); + await new Promise(resolve => setTimeout(resolve, 1000)); } - return true; } diff --git a/src/agent/modes.js b/src/agent/modes.js index f6add51..de0f09c 100644 --- a/src/agent/modes.js +++ b/src/agent/modes.js @@ -15,10 +15,10 @@ const modes = [ { name: 'self_defense', description: 'Automatically attack nearby enemies. Interrupts other actions.', + interrupts: ['all'], on: true, active: false, update: function (agent) { - if (this.active) return; const enemy = world.getNearestEntityWhere(agent.bot, entity => mc.isHostile(entity), 8); if (enemy) { agent.bot.chat(`Fighting ${enemy.name}!`); @@ -31,57 +31,53 @@ const modes = [ { name: 'hunting', description: 'Automatically hunt nearby animals when idle.', + interrupts: ['defaults'], on: true, active: false, update: function (agent) { - if (!agent.coder.executing) { - const huntable = world.getNearestEntityWhere(agent.bot, entity => mc.isHuntable(entity), 8); - if (huntable) { - execute(this, agent, async () => { - agent.bot.chat(`Hunting ${huntable.name}!`); - await skills.attackEntity(agent.bot, huntable); - }); - } + const huntable = world.getNearestEntityWhere(agent.bot, entity => mc.isHuntable(entity), 8); + if (huntable) { + execute(this, agent, async () => { + agent.bot.chat(`Hunting ${huntable.name}!`); + await skills.attackEntity(agent.bot, huntable); + }); } } }, { name: 'item_collecting', description: 'Automatically collect nearby items when idle.', + interrupts: ['followPlayer'], on: true, active: false, update: function (agent) { - if (!agent.coder.executing) { - let item = world.getNearestEntityWhere(agent.bot, entity => entity.name === 'item', 8); - if (item) { - execute(this, agent, async () => { - // wait 2 seconds for the item to settle - await new Promise(resolve => setTimeout(resolve, 2000)); - await skills.pickupNearbyItem(agent.bot); - }); - } + let item = world.getNearestEntityWhere(agent.bot, entity => entity.name === 'item', 8); + if (item) { + execute(this, agent, async () => { + // wait 2 seconds for the item to settle + await new Promise(resolve => setTimeout(resolve, 2000)); + await skills.pickupNearbyItem(agent.bot); + }); } } }, { name: 'torch_placing', description: 'Automatically place torches when idle and there are no torches nearby.', + interrupts: ['followPlayer'], on: true, active: false, update: function (agent) { - if (this.active) return; - if (!agent.coder.executing) { - // TODO: check light level instead of nearby torches, block.light is broken - const near_torch = world.getNearestBlock(agent.bot, 'torch', 8); - if (!near_torch) { - let torches = agent.bot.inventory.items().filter(item => item.name.includes('torch')); - if (torches.length > 0) { - const torch = torches[0]; - const pos = agent.bot.entity.position; - execute(this, agent, async () => { - await skills.placeBlock(agent.bot, torch.name, pos.x, pos.y, pos.z); - }); - } + // TODO: check light level instead of nearby torches, block.light is broken + const near_torch = world.getNearestBlock(agent.bot, 'torch', 8); + if (!near_torch) { + let torches = agent.bot.inventory.items().filter(item => item.name.includes('torch')); + if (torches.length > 0) { + const torch = torches[0]; + const pos = agent.bot.entity.position; + execute(this, agent, async () => { + await skills.placeBlock(agent.bot, torch.name, pos.x, pos.y, pos.z); + }); } } } @@ -89,6 +85,7 @@ const modes = [ { name: 'idle_staring', description: 'Non-functional animation to look around at entities when idle.', + interrupts: [], on: true, active: false, @@ -96,35 +93,30 @@ const modes = [ last_entity: null, next_change: 0, update: function (agent) { - if (!agent.coder.executing) { - this.active = true; - const entity = agent.bot.nearestEntity(); - let entity_in_view = entity && entity.position.distanceTo(agent.bot.entity.position) < 10 && entity.name !== 'enderman'; - if (entity_in_view && entity !== this.last_entity) { - this.staring = true; - this.last_entity = entity; - this.next_change = Date.now() + Math.random() * 1000 + 4000; - } - if (entity_in_view && this.staring) { - let isbaby = entity.type !== 'player' && entity.metadata[16]; - let height = isbaby ? entity.height/2 : entity.height; - agent.bot.lookAt(entity.position.offset(0, height, 0)); - } - if (!entity_in_view) - this.last_entity = null; - if (Date.now() > this.next_change) { - // look in random direction - this.staring = Math.random() < 0.3; - if (!this.staring) { - const yaw = Math.random() * Math.PI * 2; - const pitch = (Math.random() * Math.PI/2) - Math.PI/4; - agent.bot.look(yaw, pitch, false); - } - this.next_change = Date.now() + Math.random() * 10000 + 2000; - } + const entity = agent.bot.nearestEntity(); + let entity_in_view = entity && entity.position.distanceTo(agent.bot.entity.position) < 10 && entity.name !== 'enderman'; + if (entity_in_view && entity !== this.last_entity) { + this.staring = true; + this.last_entity = entity; + this.next_change = Date.now() + Math.random() * 1000 + 4000; + } + if (entity_in_view && this.staring) { + let isbaby = entity.type !== 'player' && entity.metadata[16]; + let height = isbaby ? entity.height/2 : entity.height; + agent.bot.lookAt(entity.position.offset(0, height, 0)); + } + if (!entity_in_view) + this.last_entity = null; + if (Date.now() > this.next_change) { + // look in random direction + this.staring = Math.random() < 0.3; + if (!this.staring) { + const yaw = Math.random() * Math.PI * 2; + const pitch = (Math.random() * Math.PI/2) - Math.PI/4; + agent.bot.look(yaw, pitch, false); + } + this.next_change = Date.now() + Math.random() * 10000 + 2000; } - else - this.active = false; } }, ]; @@ -183,12 +175,12 @@ class ModeController { } } for (let mode of this.modes_list) { - if (mode.on && !mode.paused) { + let available = mode.interrupts.includes('all') || !this.agent.coder.executing; + let interruptible = this.agent.coder.interruptible && (mode.interrupts.includes('defaults') || mode.interrupts.includes(this.agent.coder.default_name)); + if (mode.on && !mode.paused && !mode.active && (available || interruptible)) { mode.update(this.agent); - if (mode.active) { - break; - } } + if (mode.active) break; } } } From 5dd4bbb8cd1977ff0ce51b9f5aff2bf1e8ed42f7 Mon Sep 17 00:00:00 2001 From: Kolby Nottingham Date: Fri, 26 Jan 2024 14:10:09 -0800 Subject: [PATCH 2/7] newAction idle compatibility --- src/agent/coder.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/agent/coder.js b/src/agent/coder.js index 9e973c7..d1ed37a 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -93,11 +93,11 @@ export class Coder { // wrapper to prevent overlapping code generation loops await this.stop(); this.generating = true; - await this.generateCodeLoop(agent_history); + let res = await this.generateCodeLoop(agent_history); this.generating = false; + if (!res.interrupted) this.agent.bot.emit('idle'); } - async generateCodeLoop(agent_history) { let system_message = "You are a minecraft mineflayer bot that plays minecraft by writing javascript codeblocks. Given the conversation between you and the user, use the provided skills and world functions to write your code in a codeblock. Example response: ``` // your code here ``` You will then be given a response to your code. If you are satisfied with the response, respond without a codeblock in a conversational way. If something went wrong, write another codeblock and try to fix the problem."; system_message += getSkillDocs(); @@ -120,11 +120,11 @@ export class Coder { agent_history.add('system', code_return.message); agent_history.add(this.agent.name, res); this.agent.bot.chat(res); - return; + return {success: true, message: null, interrupted: false, timedout: false}; } if (failures >= 1) { agent_history.add('system', 'Action failed, agent would not write code.'); - return; + return {success: false, message: null, interrupted: false, timedout: false}; } messages.push({ role: 'system', @@ -138,14 +138,14 @@ export class Coder { const execution_file = await this.stageCode(code); if (!execution_file) { agent_history.add('system', 'Failed to stage code, something is wrong.'); - return; + return {success: false, message: null, interrupted: false, timedout: false}; } code_return = await this.execute(async ()=>{ return await execution_file.main(this.agent.bot); }); if (code_return.interrupted && !code_return.timedout) - return; + return {success: false, message: null, interrupted: true, timedout: false}; console.log(code_return.message); messages.push({ @@ -157,7 +157,7 @@ export class Coder { content: code_return.message }); } - return; + return {success: false, message: null, interrupted: false, timedout: true}; } async executeDefault(func=null, name=null, timeout=10) { @@ -196,7 +196,7 @@ export class Coder { let interrupted = this.agent.bot.interrupt_code; let timedout = this.timedout; this.clear(); - if (!interrupted) this.agent.bot.emit('idle'); + if (!interrupted && !this.generating) this.agent.bot.emit('idle'); return {success:true, message: output, interrupted, timedout}; } catch (err) { this.executing = false; @@ -208,7 +208,7 @@ export class Coder { let message = this.formatOutput(this.agent.bot) + '!!Code threw exception!! Error: ' + err; let interrupted = this.agent.bot.interrupt_code; this.clear(); - if (!interrupted) this.agent.bot.emit('idle'); + if (!interrupted && !this.generating) this.agent.bot.emit('idle'); return {success: false, message, interrupted, timedout: false}; } } From 5a27e005c7eebd8ee4bb1d79a7757285166d69ae Mon Sep 17 00:00:00 2001 From: Kolby Nottingham Date: Sat, 3 Feb 2024 08:59:07 -0800 Subject: [PATCH 3/7] execute resume wait --- src/agent/coder.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/agent/coder.js b/src/agent/coder.js index c6c65ab..becb7cb 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -168,6 +168,7 @@ export class Coder { } if (this.resume_func != null) { this.interruptible = true; + await new Promise(resolve => setTimeout(resolve, 500)); let res = await this.execute(this.resume_func, timeout); this.interruptible = false; return res; From d968f8d5b94c6c4c0d1a18a151ca290f3f577af0 Mon Sep 17 00:00:00 2001 From: Kolby Nottingham Date: Sat, 3 Feb 2024 16:19:20 -0800 Subject: [PATCH 4/7] check idle before resuming --- src/agent/coder.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/agent/coder.js b/src/agent/coder.js index becb7cb..5dc0443 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -13,9 +13,6 @@ export class Coder { this.generating = false; this.code_template = ''; this.timedout = false; - this.resume_func = null; - this.resume_name = null; - this.interruptible = false; } async load() { @@ -166,9 +163,9 @@ export class Coder { this.resume_func = func; this.resume_name = name; } - if (this.resume_func != null) { + await new Promise(resolve => setTimeout(resolve, 500)); + if (this.resume_func != null && this.agent.isIdle()) { this.interruptible = true; - await new Promise(resolve => setTimeout(resolve, 500)); let res = await this.execute(this.resume_func, timeout); this.interruptible = false; return res; From f9c41246f16924fd46987f755d2c48dccff56d94 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Mon, 5 Feb 2024 13:21:32 -0600 Subject: [PATCH 5/7] added cancelResume for failed actions --- src/agent/coder.js | 8 ++++++-- src/agent/commands/actions.js | 7 ++++--- src/agent/library/skills.js | 2 +- src/examples.json | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/agent/coder.js b/src/agent/coder.js index 6ef83e9..aca2f5c 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -163,7 +163,6 @@ export class Coder { this.resume_func = func; this.resume_name = name; } - await new Promise(resolve => setTimeout(resolve, 500)); if (this.resume_func != null && this.agent.isIdle()) { this.interruptible = true; let res = await this.execute(this.resume_func, timeout); @@ -174,6 +173,11 @@ export class Coder { } } + cancelResume() { + this.resume_func = null; + this.resume_name = null; + } + // returns {success: bool, message: string, interrupted: bool, timedout: false} async execute(func, timeout=10) { if (!this.code_template) return {success: false, message: "Code template not loaded.", interrupted: false, timedout: false}; @@ -200,7 +204,7 @@ export class Coder { } catch (err) { this.executing = false; clearTimeout(TIMEOUT); - + this.cancelResume(); console.error("Code execution triggered catch: " + err); await this.stop(); diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index 78461ff..d9b88cf 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -35,8 +35,7 @@ export const actionsList = [ perform: async function (agent) { await agent.coder.stop(); agent.coder.clear(); - agent.coder.resume_func = null; - agent.coder.resume_name = null; + agent.coder.cancelResume(); return 'Agent stopped.'; } }, @@ -126,7 +125,9 @@ export const actionsList = [ 'type': '(string) The block type to collect. Ex: !collectAllBlocks("stone")' }, perform: wrapExecution(async (agent, type) => { - await skills.collectBlock(agent.bot, type, 1); + let success = await skills.collectBlock(agent.bot, type, 1); + if (!success) + agent.coder.cancelResume(); }, 10, 'collectAllBlocks') // 10 minute timeout }, { diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index 010bc16..5c080c3 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -373,7 +373,7 @@ export async function collectBlock(bot, blockType, num=1) { break; } log(bot, `Collected ${collected} ${blockType}.`); - return true; + return collected > 0; } export async function pickupNearbyItems(bot) { diff --git a/src/examples.json b/src/examples.json index 863a731..07c30be 100644 --- a/src/examples.json +++ b/src/examples.json @@ -54,7 +54,7 @@ ], [ - {"role": "user", "content": "billybob: stop"}, + {"role": "user", "content": "abc: stop"}, {"role": "assistant", "content": "Sure. !stop"} ], From 271f5672f4f71c89c917c97c4b1815f1a7f8bc94 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Mon, 5 Feb 2024 13:48:02 -0600 Subject: [PATCH 6/7] cancelresume on death --- src/agent/agent.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/agent/agent.js b/src/agent/agent.js index 890b5ef..dfaf425 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -158,6 +158,7 @@ export class Agent { process.exit(1); }); this.bot.on('death', () => { + this.coder.cancelResume(); this.coder.stop(); }); this.bot.on('kicked', (reason) => { From 0e84991ce53b231b498f33c520d4717c693daa06 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Mon, 5 Feb 2024 14:05:52 -0600 Subject: [PATCH 7/7] sneaking in torch placing improvment --- src/agent/library/skills.js | 2 +- src/agent/modes.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index 5c080c3..d5abb0b 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -12,7 +12,7 @@ export function log(bot, message, chat=false) { async function autoLight(bot) { if (bot.modes.isOn('torch_placing') && !bot.interrupt_code) { - let nearest_torch = world.getNearestBlock(bot, 'torch', 8); + let nearest_torch = world.getNearestBlock(bot, 'torch', 6); if (!nearest_torch) { let has_torch = bot.inventory.items().find(item => item.name === 'torch'); if (has_torch) { diff --git a/src/agent/modes.js b/src/agent/modes.js index 92d7a77..78b62a6 100644 --- a/src/agent/modes.js +++ b/src/agent/modes.js @@ -85,7 +85,7 @@ const modes = [ active: false, update: function (agent) { // TODO: check light level instead of nearby torches, block.light is broken - const near_torch = world.getNearestBlock(agent.bot, 'torch', 8); + const near_torch = world.getNearestBlock(agent.bot, 'torch', 6); if (!near_torch) { let torches = agent.bot.inventory.items().filter(item => item.name.includes('torch')); if (torches.length > 0) {