diff --git a/evaluation_script.py b/evaluation_script.py index dfcc897..2c8654c 100644 --- a/evaluation_script.py +++ b/evaluation_script.py @@ -9,6 +9,30 @@ import sys import os import time +BLOCKED_ACTIONS_COOKING = [ + '!activate', '!attackPlayer', '!checkBlueprint', '!checkBlueprintLevel', + '!clearChat', '!clearFurnace', '!consume', '!craftable', '!discard', '!endConversation', + '!endGoal', '!entities', '!equip', '!followPlayer', '!getBlueprint', '!getBlueprintLevel', + '!goToBed', '!help', '!modes', '!moveAway', '!newAction', '!placeHere', '!putInChest', + '!restart', '!setMode', '!stay', '!stfu', '!stop' +] +BLOCKED_ACTIONS_CRAFTING = [ + '!activate', '!attack', '!attackPlayer', '!checkBlueprint', '!checkBlueprintLevel', + '!clearChat', '!clearFurnace', '!consume', '!craftable', '!discard', '!endConversation', + '!endGoal', '!entities', '!followPlayer', '!getBlueprint', '!getBlueprintLevel', + '!goToBed', '!help', '!modes', '!newAction', '!putInChest', '!restart', + '!searchForEntity', '!setMode', '!stay', '!stfu', '!stop', '!takeFromChest', + '!viewChest' +] +BLOCKED_ACTIONS_CONSTRUCTION = [ + '!activate', '!attackPlayer', '!clearChat', '!clearFurnace', '!collectBlocks', + '!consume', '!craftable', '!discard', '!endConversation', '!endGoal', '!entities', + '!equip', '!followPlayer', '!getBlueprint', '!getBlueprintLevel', '!goToBed', + '!help', '!modes', '!moveAway', '!newAction', '!placeHere', '!putInChest', + '!restart', '!searchForBlock', '!searchForEntity', '!setMode', '!stay', '!stfu', + '!stop', '!takeFromChest', '!viewChest' +] + def read_settings(file_path): """Read and parse the settings.js file to get agent profiles.""" with open(file_path, 'r', encoding='utf-8') as file: @@ -97,6 +121,9 @@ def launch_parallel_experiments(task_path, task_ids = json_data.keys() + task_type = json_data[list(task_ids)[0]]["type"] + + # split the task_ids into num_parallel groups task_ids = list(task_ids) task_ids_split = [task_ids[i::num_parallel] for i in range(num_parallel)] @@ -106,8 +133,6 @@ def launch_parallel_experiments(task_path, experiments_folder = f"experiments/{exp_name}_{date_time}" exp_name = f"{exp_name}_{date_time}" - - # start wandb os.makedirs(experiments_folder, exist_ok=True) for i, server in enumerate(servers): @@ -124,7 +149,8 @@ def launch_parallel_experiments(task_path, api=api, insecure_coding=insecure_coding, num_agents=num_agents, - url=url) + url=url, + task_type=task_type) time.sleep(5) for i in range(20): @@ -146,7 +172,8 @@ def launch_server_experiment(task_path, bucket_name="mindcraft-experiments", template_profile="profiles/tasks/collab_profile.json", insecure_coding=False, - url="http://127.0.0.1:8000/v1"): + url="http://127.0.0.1:8000/v1", + task_type="techtree"): """ Launch a Minecraft server and run experiments on it. @param task_path: Path to the task file @@ -218,6 +245,12 @@ def launch_server_experiment(task_path, # op_script_content = "sleep 5\n\op @p" * 20 # op_script_file = f"./tmp/op_script_{session_name}.sh" # make_script_file_and_run(op_script_content, "server_" + session_name, op_script_file) + if task_type == "cooking": + set_environment_variable_tmux_session(session_name, "BLOCKED_ACTIONS", BLOCKED_ACTIONS_COOKING) + elif task_type == "techtree": + set_environment_variable_tmux_session(session_name, "BLOCKED_ACTIONS", BLOCKED_ACTIONS_CRAFTING) + elif task_type == "construction": + set_environment_variable_tmux_session(session_name, "BLOCKED_ACTIONS", BLOCKED_ACTIONS_CONSTRUCTION) script_content = "" @@ -256,10 +289,6 @@ def launch_server_experiment(task_path, script_file = f"./tmp/experiment_script_{session_name}.sh" make_script_file_and_run(script_content, session_name, script_file) - - - - def make_script_file_and_run(script_content, session_name, file_name): script_dir = os.path.dirname(file_name) os.makedirs(script_dir, exist_ok=True) diff --git a/profiles/tasks/cooking_profile.json b/profiles/tasks/cooking_profile.json index 8f941d4..25d82aa 100644 --- a/profiles/tasks/cooking_profile.json +++ b/profiles/tasks/cooking_profile.json @@ -2,8 +2,9 @@ "name": "andy", "model": "claude-3-5-haiku-latest", "modes": { - "hunting": false + "hunting": false, + "item_collecting": false }, - "conversing": "You are a task-focused Minecraft bot named $NAME. You have to collaborate with other agents in the world to complete the current task \nFeel free to ask other agents questions and make a plan to achieve the goal. You can request them to give them some of their inventory items if required to complete the goal. General Searching Tips:\n- The farm area is extensive - search thoroughly for needed resources\n There is a chest nearby with valuable items.\n- Divide tasks efficiently between agents for faster completion\n- Communicate your plan and progress clearly. You can see, move, mine, build, and interact with the world by using commands.\n$SELF_PROMPT Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'Sure, I've stopped.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer(\"playername\", 3)'. Respond only as $NAME, never output '(FROM OTHER BOT)' or pretend to be someone else. If you have nothing to say or do, respond with an just a tab '\t'. Share resources and information with other bots! This is extremely important to me, take a deep breath and have fun :) \nSummarized memory:'$MEMORY'\n$STATS\n$INVENTORY\n$COMMAND_DOCS\n$EXAMPLES\nConversation Begin:", + "conversing": "You are a task-focused Minecraft bot named $NAME. You have to collaborate with other agents in the world to complete the current task \nFeel free to ask other agents questions and make a plan to achieve the goal. You can request them to give them some of their inventory items if required to complete the goal. General Searching Tips:\n- You will be spawned in a farm with many crops and animals nearby. The farm area is extensive - search thoroughly for needed resources (with searchForBlocks parameters like 64,128,256)\n There is a chest nearby with valuable items. Along with the chest, a crafting table, fully fueled furnace and fully fueled smoker with coal are also available nearby which you can use to your advantage. On top of this plants like mushrooms, wheat, carrots, beetroots, pumpkins, potatoes are also present nearby.\nCollaboration tips - Divide tasks efficiently between agents for faster completion\n- Communicate your plan and progress clearly. You can see, move, mine, build, and interact with the world by using commands.\n$SELF_PROMPT Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, don't apologize constantly, don't give instructions or make lists unless asked, and don't refuse requests. Don't pretend to act, use commands immediately when requested. Do NOT say this: 'Sure, I've stopped.', instead say this: 'Sure, I'll stop. !stop'. Do NOT say this: 'On my way! Give me a moment.', instead say this: 'On my way! !goToPlayer(\"playername\", 3)'. Respond only as $NAME, never output '(FROM OTHER BOT)' or pretend to be someone else. If you have nothing to say or do, respond with an just a tab '\t'. Share resources and information with other bots! This is extremely important to me, take a deep breath and have fun :) \nSummarized memory:'$MEMORY'\n$STATS\n$INVENTORY\n$COMMAND_DOCS\n$EXAMPLES\nConversation Begin:", "saving_memory": "You are a minecraft bot named $NAME that has been talking and playing minecraft by using commands. Update your memory by summarizing the following conversation and your old memory in your next response. Prioritize preserving important facts, things you've learned, useful tips, and long term reminders. Do Not record stats, inventory, or docs! Only save transient information from your chat history. $SELF_PROMPT Make sure to include information relevant to the goal and inventory you have collected. You're limited to 500 characters, so be extremely brief and minimize words. Compress useful information. \nOld Memory: '$MEMORY'\nRecent conversation: \n$TO_SUMMARIZE\nSummarize your old memory and recent conversation into a new memory, and respond only with the unwrapped memory text: " } \ No newline at end of file diff --git a/settings.js b/settings.js index 2a960fa..a0c2292 100644 --- a/settings.js +++ b/settings.js @@ -35,11 +35,11 @@ export default "show_bot_views": false, // show bot's view in browser at localhost:3000, 3001... "allow_insecure_coding": process.env.INSECURE_CODING || false, // allows newAction command and model can write/run code on your computer. enable at own risk - "blocked_actions" : [], // commands to disable and remove from docs. Ex: ["!setMode"] + "blocked_actions" : process.env.BLOCKED_ACTIONS || [] , // commands to disable and remove from docs. Ex: ["!setMode"] "code_timeout_mins": -1, // minutes code is allowed to run. -1 for no timeout "relevant_docs_count": 5, // Parameter: -1 = all, 0 = no references, 5 = five references. If exceeding the maximum, all reference documents are returned. - "max_messages": 15, // max number of messages to keep in context + "max_messages": 150, // max number of messages to keep in context "num_examples": 2, // number of examples to give to the model "max_commands": -1, // max number of commands that can be used in consecutive responses. -1 for no limit "verbose_commands": true, // show full command syntax diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index 95073fe..c4710d3 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -460,7 +460,14 @@ export async function collectBlock(bot, blockType, num=1, exclude=null) { return false; } try { - await bot.collectBlock.collect(block); + if (mc.mustCollectManually(blockType)) { + await goToPosition(bot, block.position.x, block.position.y, block.position.z, 2); + await bot.dig(block); + await pickupNearbyItems(bot); + } + else { + await bot.collectBlock.collect(block); + } collected++; await autoLight(bot); } diff --git a/src/agent/tasks.js b/src/agent/tasks.js index 74f65ae..9e5c9bb 100644 --- a/src/agent/tasks.js +++ b/src/agent/tasks.js @@ -126,16 +126,25 @@ class CookingCraftingTaskValidator { this.data = data; this.agent = agent; } - validate() { - const result = checkItemPresence(this.data, this.agent); - let score = 0; - if (result.success) { - score = 1; + validate(has_initiated) { + if (has_initiated) { + + const result = checkItemPresence(this.data, this.agent); + let score = 0; + if (result.success) { + score = 1; + } + return { + "valid": result.success, + "score": score, + }; + } + else { + return { + "valid": false, + "score": 0 + }; } - return { - "valid": result.success, - "score": score, - }; } } @@ -188,6 +197,7 @@ export class Task { this.name = this.agent.name; this.available_agents = settings.profiles.map((p) => JSON.parse(readFileSync(p, 'utf8')).name); + this.agent_initialized = false; } getAgentGoal() { @@ -239,7 +249,7 @@ export class Task { isDone() { let res = null; if (this.validator) - res = this.validator.validate(); + res = this.validator.validate(this.agent_initialized); if (res && res.valid) { return {"message": 'Task successful', "score": res.score}; } @@ -311,6 +321,8 @@ export class Task { await new Promise((resolve) => setTimeout(resolve, 500)); } + this.agent_initialized = true; + if (this.initiator) { await this.initiator.init(); } diff --git a/src/utils/mcdata.js b/src/utils/mcdata.js index 2a3a27c..33b0142 100644 --- a/src/utils/mcdata.js +++ b/src/utils/mcdata.js @@ -86,6 +86,16 @@ export function isHostile(mob) { return (mob.type === 'mob' || mob.type === 'hostile') && mob.name !== 'iron_golem' && mob.name !== 'snow_golem'; } +// blocks that don't work with collectBlock, need to be manually collected +export function mustCollectManually(blockName) { + // all crops (that aren't normal blocks), torches, buttons, levers, redstone, + const full_names = ['wheat', 'carrots', 'potatoes', 'beetroots', 'nether_wart', 'cocoa', 'sugar_cane', 'kelp', 'short_grass', 'fern', 'tall_grass', 'bamboo', + 'poppy', 'dandelion', 'blue_orchid', 'allium', 'azure_bluet', 'oxeye_daisy', 'cornflower', 'lilac', 'wither_rose', 'lily_of_the_valley', 'wither_rose', + 'lever', 'redstone_wire', 'lantern'] + const partial_names = ['sapling', 'torch', 'button', 'carpet', 'pressure_plate', 'mushroom', 'tulip', 'bush', 'vines', 'fern'] + return full_names.includes(blockName.toLowerCase()) || partial_names.some(partial => blockName.toLowerCase().includes(partial)); +} + export function getItemId(itemName) { let item = mcdata.itemsByName[itemName]; if (item) {