From d399f8e214afc23db87d36561634b719d5d48c0e Mon Sep 17 00:00:00 2001 From: Isadora White Date: Fri, 14 Mar 2025 18:49:33 -0700 Subject: [PATCH 1/5] blocked actions --- evaluation_script.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/evaluation_script.py b/evaluation_script.py index dfcc897..b6cdc86 100644 --- a/evaluation_script.py +++ b/evaluation_script.py @@ -9,6 +9,10 @@ import sys import os import time +BLOCKED_ACTIONS_COOKING = [] +BLOCKED_ACTIONS_CRAFTING = [] +BLOCKED_ACTIONS_CONSTRUCTION = [] + def read_settings(file_path): """Read and parse the settings.js file to get agent profiles.""" with open(file_path, 'r', encoding='utf-8') as file: @@ -97,6 +101,9 @@ def launch_parallel_experiments(task_path, task_ids = json_data.keys() + task_type = json_data[list(task_ids)[0]]["type"] + + # split the task_ids into num_parallel groups task_ids = list(task_ids) task_ids_split = [task_ids[i::num_parallel] for i in range(num_parallel)] @@ -106,8 +113,6 @@ def launch_parallel_experiments(task_path, experiments_folder = f"experiments/{exp_name}_{date_time}" exp_name = f"{exp_name}_{date_time}" - - # start wandb os.makedirs(experiments_folder, exist_ok=True) for i, server in enumerate(servers): @@ -124,7 +129,8 @@ def launch_parallel_experiments(task_path, api=api, insecure_coding=insecure_coding, num_agents=num_agents, - url=url) + url=url, + task_type=task_type) time.sleep(5) for i in range(20): @@ -146,7 +152,8 @@ def launch_server_experiment(task_path, bucket_name="mindcraft-experiments", template_profile="profiles/tasks/collab_profile.json", insecure_coding=False, - url="http://127.0.0.1:8000/v1"): + url="http://127.0.0.1:8000/v1", + task_type="techtree"): """ Launch a Minecraft server and run experiments on it. @param task_path: Path to the task file @@ -218,6 +225,12 @@ def launch_server_experiment(task_path, # op_script_content = "sleep 5\n\op @p" * 20 # op_script_file = f"./tmp/op_script_{session_name}.sh" # make_script_file_and_run(op_script_content, "server_" + session_name, op_script_file) + if task_type == "cooking": + set_environment_variable_tmux_session(session_name, "BLOCKED_ACTIONS", BLOCKED_ACTIONS_COOKING) + elif task_type == "techtree": + set_environment_variable_tmux_session(session_name, "BLOCKED_ACTIONS", BLOCKED_ACTIONS_CRAFTING) + elif task_type == "construction": + set_environment_variable_tmux_session(session_name, "BLOCKED_ACTIONS", BLOCKED_ACTIONS_CONSTRUCTION) script_content = "" @@ -256,10 +269,6 @@ def launch_server_experiment(task_path, script_file = f"./tmp/experiment_script_{session_name}.sh" make_script_file_and_run(script_content, session_name, script_file) - - - - def make_script_file_and_run(script_content, session_name, file_name): script_dir = os.path.dirname(file_name) os.makedirs(script_dir, exist_ok=True) From 125aa73d6c961e96f082e313a70153699d867632 Mon Sep 17 00:00:00 2001 From: Isadora White Date: Fri, 14 Mar 2025 18:51:41 -0700 Subject: [PATCH 2/5] adding blocked actions --- evaluation_script.py | 26 +++++++++++++++++++++++--- settings.js | 4 ++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/evaluation_script.py b/evaluation_script.py index b6cdc86..2c8654c 100644 --- a/evaluation_script.py +++ b/evaluation_script.py @@ -9,9 +9,29 @@ import sys import os import time -BLOCKED_ACTIONS_COOKING = [] -BLOCKED_ACTIONS_CRAFTING = [] -BLOCKED_ACTIONS_CONSTRUCTION = [] +BLOCKED_ACTIONS_COOKING = [ + '!activate', '!attackPlayer', '!checkBlueprint', '!checkBlueprintLevel', + '!clearChat', '!clearFurnace', '!consume', '!craftable', '!discard', '!endConversation', + '!endGoal', '!entities', '!equip', '!followPlayer', '!getBlueprint', '!getBlueprintLevel', + '!goToBed', '!help', '!modes', '!moveAway', '!newAction', '!placeHere', '!putInChest', + '!restart', '!setMode', '!stay', '!stfu', '!stop' +] +BLOCKED_ACTIONS_CRAFTING = [ + '!activate', '!attack', '!attackPlayer', '!checkBlueprint', '!checkBlueprintLevel', + '!clearChat', '!clearFurnace', '!consume', '!craftable', '!discard', '!endConversation', + '!endGoal', '!entities', '!followPlayer', '!getBlueprint', '!getBlueprintLevel', + '!goToBed', '!help', '!modes', '!newAction', '!putInChest', '!restart', + '!searchForEntity', '!setMode', '!stay', '!stfu', '!stop', '!takeFromChest', + '!viewChest' +] +BLOCKED_ACTIONS_CONSTRUCTION = [ + '!activate', '!attackPlayer', '!clearChat', '!clearFurnace', '!collectBlocks', + '!consume', '!craftable', '!discard', '!endConversation', '!endGoal', '!entities', + '!equip', '!followPlayer', '!getBlueprint', '!getBlueprintLevel', '!goToBed', + '!help', '!modes', '!moveAway', '!newAction', '!placeHere', '!putInChest', + '!restart', '!searchForBlock', '!searchForEntity', '!setMode', '!stay', '!stfu', + '!stop', '!takeFromChest', '!viewChest' +] def read_settings(file_path): """Read and parse the settings.js file to get agent profiles.""" diff --git a/settings.js b/settings.js index 2a960fa..a0c2292 100644 --- a/settings.js +++ b/settings.js @@ -35,11 +35,11 @@ export default "show_bot_views": false, // show bot's view in browser at localhost:3000, 3001... "allow_insecure_coding": process.env.INSECURE_CODING || false, // allows newAction command and model can write/run code on your computer. enable at own risk - "blocked_actions" : [], // commands to disable and remove from docs. Ex: ["!setMode"] + "blocked_actions" : process.env.BLOCKED_ACTIONS || [] , // commands to disable and remove from docs. Ex: ["!setMode"] "code_timeout_mins": -1, // minutes code is allowed to run. -1 for no timeout "relevant_docs_count": 5, // Parameter: -1 = all, 0 = no references, 5 = five references. If exceeding the maximum, all reference documents are returned. - "max_messages": 15, // max number of messages to keep in context + "max_messages": 150, // max number of messages to keep in context "num_examples": 2, // number of examples to give to the model "max_commands": -1, // max number of commands that can be used in consecutive responses. -1 for no limit "verbose_commands": true, // show full command syntax From c5b834597ab17b21ebe6d7dd27b1628d4566bd3d Mon Sep 17 00:00:00 2001 From: Ayush Maniar Date: Sat, 15 Mar 2025 14:27:56 -0700 Subject: [PATCH 3/5] Turned off item_collecting for cooking_profile.json --- profiles/tasks/cooking_profile.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/profiles/tasks/cooking_profile.json b/profiles/tasks/cooking_profile.json index 8f941d4..25d82aa 100644 --- a/profiles/tasks/cooking_profile.json +++ b/profiles/tasks/cooking_profile.json @@ -2,8 +2,9 @@ "name": "andy", "model": "claude-3-5-haiku-latest", "modes": { - "hunting": false + "hunting": false, + "item_collecting": false }, - "conversing": "You are a task-focused Minecraft bot named $NAME. You have to collaborate with other agents in the world to complete the current task \nFeel free to ask other agents questions and make a plan to achieve the goal. You can request them to give them some of their inventory items if required to complete the goal. General Searching Tips:\n- The farm area is extensive - search thoroughly for needed resources\n There is a chest nearby with valuable items.\n- Divide tasks efficiently between agents for faster completion\n- Communicate your plan and progress clearly. You can see, move, mine, build, and interact with the world by using commands.\n$SELF_PROMPT Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'Sure, I've stopped.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer(\"playername\", 3)'. Respond only as $NAME, never output '(FROM OTHER BOT)' or pretend to be someone else. If you have nothing to say or do, respond with an just a tab '\t'. Share resources and information with other bots! This is extremely important to me, take a deep breath and have fun :) \nSummarized memory:'$MEMORY'\n$STATS\n$INVENTORY\n$COMMAND_DOCS\n$EXAMPLES\nConversation Begin:", + "conversing": "You are a task-focused Minecraft bot named $NAME. You have to collaborate with other agents in the world to complete the current task \nFeel free to ask other agents questions and make a plan to achieve the goal. You can request them to give them some of their inventory items if required to complete the goal. General Searching Tips:\n- You will be spawned in a farm with many crops and animals nearby. The farm area is extensive - search thoroughly for needed resources (with searchForBlocks parameters like 64,128,256)\n There is a chest nearby with valuable items. Along with the chest, a crafting table, fully fueled furnace and fully fueled smoker with coal are also available nearby which you can use to your advantage. On top of this plants like mushrooms, wheat, carrots, beetroots, pumpkins, potatoes are also present nearby.\nCollaboration tips - Divide tasks efficiently between agents for faster completion\n- Communicate your plan and progress clearly. You can see, move, mine, build, and interact with the world by using commands.\n$SELF_PROMPT Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'Sure, I've stopped.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer(\"playername\", 3)'. Respond only as $NAME, never output '(FROM OTHER BOT)' or pretend to be someone else. If you have nothing to say or do, respond with an just a tab '\t'. Share resources and information with other bots! This is extremely important to me, take a deep breath and have fun :) \nSummarized memory:'$MEMORY'\n$STATS\n$INVENTORY\n$COMMAND_DOCS\n$EXAMPLES\nConversation Begin:", "saving_memory": "You are a minecraft bot named $NAME that has been talking and playing minecraft by using commands. Update your memory by summarizing the following conversation and your old memory in your next response. Prioritize preserving important facts, things you've learned, useful tips, and long term reminders. Do Not record stats, inventory, or docs! Only save transient information from your chat history. $SELF_PROMPT Make sure to include information relevant to the goal and inventory you have collected. You're limited to 500 characters, so be extremely brief and minimize words. Compress useful information. \nOld Memory: '$MEMORY'\nRecent conversation: \n$TO_SUMMARIZE\nSummarize your old memory and recent conversation into a new memory, and respond only with the unwrapped memory text: " } \ No newline at end of file From 4b1e02fc95fcce62896079fe4140e04b4bf4a333 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Sat, 15 Mar 2025 16:54:16 -0500 Subject: [PATCH 4/5] add manual collection for crops/other weird blocks --- src/agent/library/skills.js | 9 ++++++++- src/utils/mcdata.js | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index 95073fe..c4710d3 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -460,7 +460,14 @@ export async function collectBlock(bot, blockType, num=1, exclude=null) { return false; } try { - await bot.collectBlock.collect(block); + if (mc.mustCollectManually(blockType)) { + await goToPosition(bot, block.position.x, block.position.y, block.position.z, 2); + await bot.dig(block); + await pickupNearbyItems(bot); + } + else { + await bot.collectBlock.collect(block); + } collected++; await autoLight(bot); } diff --git a/src/utils/mcdata.js b/src/utils/mcdata.js index 2a3a27c..33b0142 100644 --- a/src/utils/mcdata.js +++ b/src/utils/mcdata.js @@ -86,6 +86,16 @@ export function isHostile(mob) { return (mob.type === 'mob' || mob.type === 'hostile') && mob.name !== 'iron_golem' && mob.name !== 'snow_golem'; } +// blocks that don't work with collectBlock, need to be manually collected +export function mustCollectManually(blockName) { + // all crops (that aren't normal blocks), torches, buttons, levers, redstone, + const full_names = ['wheat', 'carrots', 'potatoes', 'beetroots', 'nether_wart', 'cocoa', 'sugar_cane', 'kelp', 'short_grass', 'fern', 'tall_grass', 'bamboo', + 'poppy', 'dandelion', 'blue_orchid', 'allium', 'azure_bluet', 'oxeye_daisy', 'cornflower', 'lilac', 'wither_rose', 'lily_of_the_valley', 'wither_rose', + 'lever', 'redstone_wire', 'lantern'] + const partial_names = ['sapling', 'torch', 'button', 'carpet', 'pressure_plate', 'mushroom', 'tulip', 'bush', 'vines', 'fern'] + return full_names.includes(blockName.toLowerCase()) || partial_names.some(partial => blockName.toLowerCase().includes(partial)); +} + export function getItemId(itemName) { let item = mcdata.itemsByName[itemName]; if (item) { From 7c96cca59cb64be74e60dec743a075a6cbf06db2 Mon Sep 17 00:00:00 2001 From: Ayush Maniar Date: Sat, 15 Mar 2025 22:35:23 -0700 Subject: [PATCH 5/5] Bug fix for validator execution before the bot initialization --- src/agent/tasks.js | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/agent/tasks.js b/src/agent/tasks.js index 74f65ae..9e5c9bb 100644 --- a/src/agent/tasks.js +++ b/src/agent/tasks.js @@ -126,16 +126,25 @@ class CookingCraftingTaskValidator { this.data = data; this.agent = agent; } - validate() { - const result = checkItemPresence(this.data, this.agent); - let score = 0; - if (result.success) { - score = 1; + validate(has_initiated) { + if (has_initiated) { + + const result = checkItemPresence(this.data, this.agent); + let score = 0; + if (result.success) { + score = 1; + } + return { + "valid": result.success, + "score": score, + }; + } + else { + return { + "valid": false, + "score": 0 + }; } - return { - "valid": result.success, - "score": score, - }; } } @@ -188,6 +197,7 @@ export class Task { this.name = this.agent.name; this.available_agents = settings.profiles.map((p) => JSON.parse(readFileSync(p, 'utf8')).name); + this.agent_initialized = false; } getAgentGoal() { @@ -239,7 +249,7 @@ export class Task { isDone() { let res = null; if (this.validator) - res = this.validator.validate(); + res = this.validator.validate(this.agent_initialized); if (res && res.valid) { return {"message": 'Task successful', "score": res.score}; } @@ -311,6 +321,8 @@ export class Task { await new Promise((resolve) => setTimeout(resolve, 500)); } + this.agent_initialized = true; + if (this.initiator) { await this.initiator.init(); }