From 6b4b895cc1739b3271e7299d781482d93915d64d Mon Sep 17 00:00:00 2001 From: Isadora White Date: Mon, 26 May 2025 16:46:42 -0700 Subject: [PATCH 01/13] human ai tasks --- minecollab.md | 2 + ..._three_agents.pdf => church_blueprint.pdf} | Bin .../human_ai/1_agent_1_human.json | 34 + .../human_ai_tasks/1_agent_1_human.json | 762 ------------------ tasks/running_human_ai.md | 20 + 5 files changed, 56 insertions(+), 762 deletions(-) rename tasks/construction_tasks/{church_three_agents.pdf => church_blueprint.pdf} (100%) create mode 100644 tasks/cooking_tasks/human_ai/1_agent_1_human.json create mode 100644 tasks/running_human_ai.md diff --git a/minecollab.md b/minecollab.md index 019d8b1..551c3b7 100644 --- a/minecollab.md +++ b/minecollab.md @@ -41,6 +41,8 @@ You can view the crafting task in action [here](https://www.youtube.com/shorts/V ## Installation +You **DO NOT** need Linux to run this, you can run on Windows with the --no-launch-world flag and by installing git bash. + Please follow the installation docs in the README to install mindcraft. You can create a docker image using the Dockerfile. Download the relevant task files and server data files, you can find the link [here](https://drive.google.com/drive/folders/1XygbitBBTsNO6q_doEiZHmdETpnyRmCS). The tasks files are for specifying the tasks to run and the server data is for allowing the models to launch the task in the correct world automatically. **Unzip the server_data.zip in the base `tasks/` folder**. diff --git a/tasks/construction_tasks/church_three_agents.pdf b/tasks/construction_tasks/church_blueprint.pdf similarity index 100% rename from tasks/construction_tasks/church_three_agents.pdf rename to tasks/construction_tasks/church_blueprint.pdf diff --git a/tasks/cooking_tasks/human_ai/1_agent_1_human.json b/tasks/cooking_tasks/human_ai/1_agent_1_human.json new file mode 100644 index 0000000..0212f7a --- /dev/null +++ b/tasks/cooking_tasks/human_ai/1_agent_1_human.json @@ -0,0 +1,34 @@ +{ + "multiagent_cooking_bread_golden_apple": { + "type": "cooking", + "recipes": { + "bread": [ + "Step 1: Go to the farm and collect 3 wheat.", + "Step 2: Go to the crafting table and use the wheat to craft bread." + ], + "golden_apple": [ + "Step 1: Get 1 apple and 8 gold ingots from your inventory or other bots.", + "Step 2: Go to the crafting table and surround the apple with the gold ingots to create a golden apple." + ] + }, + "agent_count": 2, + "target": { + "bread": 1, + "golden_apple": 1 + }, + "initial_inventory": { + "0": { + "gold_ingot": 4, + "apple": 1 + }, + "1": { + "gold_ingot": 4 + } + }, + "goal": { + "0": "Collaborate with other agents around you to make bread, golden_apple, The recipes are as follows:\nRecipe for bread:\n['Step 1: Go to the farm and collect 3 wheat.', 'Step 2: Go to the crafting table and use the wheat to craft bread.']\nRecipe for golden_apple:\n['Step 1: Get 1 apple and 8 gold ingots from your inventory or other bots.', 'Step 2: Go to the crafting table and surround the apple with the gold ingots to create a golden apple.']\n", + "1": "Collaborate with other agents around you to make bread, golden_apple, The recipes are as follows:\nRecipe for bread:\n['Step 1: Go to the farm and collect 3 wheat.', 'Step 2: Go to the crafting table and use the wheat to craft bread.']\nRecipe for golden_apple:\n['Step 1: Get 1 apple and 8 gold ingots from your inventory or other bots.', 'Step 2: Go to the crafting table and surround the apple with the gold ingots to create a golden apple.']\n" + }, + "conversation": "Let's collaborate to make bread, golden_apple, " + } +} \ No newline at end of file diff --git a/tasks/crafting_tasks/human_ai_tasks/1_agent_1_human.json b/tasks/crafting_tasks/human_ai_tasks/1_agent_1_human.json index e539510..cf01d07 100644 --- a/tasks/crafting_tasks/human_ai_tasks/1_agent_1_human.json +++ b/tasks/crafting_tasks/human_ai_tasks/1_agent_1_human.json @@ -27,767 +27,5 @@ "izzycw" ], "requires_ctable": false - }, - "multiagent_crafting_lime_wool_partial_plan__depth_0": { - "goal": "Collaborate with other agents to craft an lime_wool", - "conversation": "Let's work together to craft an lime_wool.", - "initial_inventory": { - "0": { - "lime_dye": 1 - }, - "1": { - "black_wool": 1 - } - }, - "agent_count": 1, - "target": "lime_wool", - "number_of_target": 1, - "type": "techtree", - "max_depth": 2, - "depth": 0, - "timeout": 300, - "blocked_actions": { - "0": [], - "1": [ - "!getCraftingPlan" - ] - }, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "missing_items": [], - "requires_ctable": false - }, - "multiagent_crafting_purple_banner_full_plan_requires_ctable__depth_0": { - "goal": "Collaborate with other agents to craft an purple_banner", - "conversation": "Let's work together to craft an purple_banner.", - "initial_inventory": { - "0": { - "purple_wool": 4, - "stick": 1 - }, - "1": { - "purple_wool": 3, - "crafting_table": 1 - } - }, - "agent_count": 1, - "target": "purple_banner", - "number_of_target": 1, - "type": "techtree", - "max_depth": 3, - "depth": 0, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_soul_campfire_partial_plan_requires_ctable__depth_0": { - "goal": "Collaborate with other agents to craft an soul_campfire", - "conversation": "Let's work together to craft an soul_campfire.", - "initial_inventory": { - "0": { - "oak_planks": 2, - "soul_sand": 1, - "dark_oak_log": 2 - }, - "1": { - "oak_planks": 1, - "dark_oak_log": 1, - "crafting_table": 1 - } - }, - "agent_count": 2, - "target": "soul_campfire", - "number_of_target": 1, - "type": "techtree", - "max_depth": 2, - "depth": 0, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [ - "!getCraftingPlan" - ], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_bookshelf_full_plan_requires_ctable__depth_0": { - "goal": "Collaborate with other agents to craft a bookshelf", - "conversation": "Let's work together to craft a bookshelf.", - "initial_inventory": { - "0": { - "oak_planks": 4, - "book": 2 - }, - "1": { - "oak_planks": 2, - "book": 1, - "crafting_table": 1 - } - }, - "agent_count": 1, - "target": "bookshelf", - "number_of_target": 1, - "type": "techtree", - "max_depth": 1, - "depth": 0, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_compass_partial_plan_requires_ctable__depth_0": { - "goal": "Collaborate with other agents to craft a compass", - "conversation": "Let's work together to craft a compass.", - "initial_inventory": { - "0": { - "iron_ingot": 2 - }, - "1": { - "iron_ingot": 2, - "redstone": 1, - "crafting_table": 1 - } - }, - "agent_count": 1, - "target": "compass", - "number_of_target": 1, - "type": "techtree", - "max_depth": 2, - "depth": 0, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [ - "!getCraftingPlan" - ], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_fishing_rod_full_plan_requires_ctable__depth_1": { - "goal": "Collaborate with other agents to craft a fishing_rod", - "conversation": "Let's work together to craft a fishing_rod.", - "initial_inventory": { - "0": { - "string": 1, - "oak_planks": 2 - }, - "1": { - "string": 1, - "crafting_table": 1 - } - }, - "agent_count": 1, - "target": "fishing_rod", - "number_of_target": 1, - "type": "techtree", - "max_depth": 3, - "depth": 1, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_cake_partial_plan_requires_ctable__depth_0": { - "goal": "Collaborate with other agents to craft a cake", - "conversation": "Let's work together to craft a cake.", - "initial_inventory": { - "0": { - "wheat": 2, - "sugar": 1, - "egg": 1 - }, - "1": { - "wheat": 1, - "milk_bucket": 2, - "crafting_table": 1 - } - }, - "agent_count": 1, - "target": "cake", - "number_of_target": 1, - "type": "techtree", - "max_depth": 3, - "depth": 0, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [ - "!getCraftingPlan" - ], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_golden_carrot_full_plan_requires_ctable__depth_0": { - "goal": "Collaborate with other agents to craft a golden_carrot", - "conversation": "Let's work together to craft a golden_carrot.", - "initial_inventory": { - "0": { - "gold_nugget": 5, - "carrot": 1 - }, - "1": { - "gold_nugget": 3, - "crafting_table": 1 - } - }, - "agent_count": 1, - "target": "golden_carrot", - "number_of_target": 1, - "type": "techtree", - "max_depth": 2, - "depth": 0, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_map_partial_plan_requires_ctable__depth_0": { - "goal": "Collaborate with other agents to craft a map", - "conversation": "Let's work together to craft a map.", - "initial_inventory": { - "0": { - "paper": 5 - }, - "1": { - "paper": 3, - "compass": 1, - "crafting_table": 1 - } - }, - "agent_count": 1, - "target": "map", - "number_of_target": 1, - "type": "techtree", - "max_depth": 1, - "depth": 0, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [ - "!getCraftingPlan" - ], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_blue_wool_full_plan__depth_0": { - "goal": "Collaborate with other agents to craft blue_wool", - "conversation": "Let's work together to craft blue_wool.", - "initial_inventory": { - "0": { - "blue_dye": 1 - }, - "1": { - "white_wool": 1 - } - }, - "agent_count": 2, - "target": "blue_wool", - "number_of_target": 1, - "type": "techtree", - "max_depth": 1, - "depth": 0, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [], - "1": [] - }, - "missing_items": [], - "requires_ctable": false - }, - "multiagent_crafting_lime_wool_partial_plan__depth_2": { - "goal": "Collaborate with other agents to craft lime_wool", - "conversation": "Let's work together to craft lime_wool.", - "initial_inventory": { - "0": { - "green_dye": 1 - }, - "1": { - "white_wool": 1, - "bone_meal": 1 - } - }, - "agent_count": 2, - "target": "lime_wool", - "number_of_target": 1, - "type": "techtree", - "max_depth": 2, - "depth": 1, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [ - "!getCraftingPlan" - ], - "1": [] - }, - "missing_items": [], - "requires_ctable": false - }, - "multiagent_crafting_magenta_wool_full_plan__depth_2": { - "goal": "Collaborate with other agents to craft magenta_wool", - "conversation": "Let's work together to craft magenta_wool.", - "initial_inventory": { - "0": { - "rose_red": 1, - "lapis_lazuli": 1 - }, - "1": { - "white_wool": 1, - "bone_meal": 1 - } - }, - "agent_count": 1, - "target": "magenta_wool", - "number_of_target": 1, - "type": "techtree", - "max_depth": 3, - "depth": 2, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [], - "1": [] - }, - "missing_items": [], - "requires_ctable": false - }, - "multiagent_crafting_chest_full_plan_requires_ctable__depth_1": { - "goal": "Collaborate with other agents to craft a chest", - "conversation": "Let's work together to craft a chest.", - "initial_inventory": { - "0": { - "oak_log": 1 - }, - "1": { - "oak_planks": 4, - "crafting_table": 1 - } - }, - "agent_count": 2, - "target": "chest", - "number_of_target": 1, - "type": "techtree", - "max_depth": 1, - "depth": 1, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_barrel_partial_plan_requires_ctable__depth_1": { - "goal": "Collaborate with other agents to craft a barrel", - "conversation": "Let's work together to craft a barrel.", - "initial_inventory": { - "0": { - "spruce_planks": 3, - "crafting_table": 1 - }, - "1": { - "spruce_planks": 3, - "wooden_slab": 1 - } - }, - "agent_count": 2, - "target": "barrel", - "number_of_target": 1, - "type": "techtree", - "max_depth": 2, - "depth": 1, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [ - "!getCraftingPlan" - ], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_lectern_full_plan_requires_ctable__depth_2": { - "goal": "Collaborate with other agents to craft a lectern", - "conversation": "Let's work together to craft a lectern.", - "initial_inventory": { - "0": { - "birch_slab": 5, - "crafting_table": 1 - }, - "1": { - "birch_log": 2, - "book": 3 - } - }, - "agent_count": 1, - "target": "lectern", - "number_of_target": 1, - "type": "techtree", - "max_depth": 2, - "depth": 2, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_clock_partial_plan_requires_ctable__depth_0": { - "goal": "Collaborate with other agents to craft a clock", - "conversation": "Let's work together to craft a clock.", - "initial_inventory": { - "0": { - "gold_ingot": 2 - }, - "1": { - "gold_ingot": 2, - "redstone": 1, - "crafting_table": 1 - } - }, - "agent_count": 1, - "target": "clock", - "number_of_target": 1, - "type": "techtree", - "max_depth": 2, - "depth": 0, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [ - "!getCraftingPlan" - ], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_firework_rocket_partial_plan__depth_0": { - "goal": "Collaborate with other agents to craft firework_rocket", - "conversation": "Let's work together to craft firework_rocket.", - "initial_inventory": { - "0": { - "paper": 1 - }, - "1": { - "gunpowder": 3 - } - }, - "agent_count": 2, - "target": "firework_rocket", - "number_of_target": 1, - "type": "techtree", - "max_depth": 2, - "depth": 0, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [ - "!getCraftingPlan" - ], - "1": [] - }, - "missing_items": [], - "requires_ctable": false - }, - "multiagent_crafting_enchanting_table_partial_plan_requires_ctable__depth_0": { - "goal": "Collaborate with other agents to craft an enchanting_table", - "conversation": "Let's work together to craft an enchanting_table.", - "initial_inventory": { - "0": { - "diamond": 2, - "obsidian": 2, - "crafting_table": 1 - }, - "1": { - "obsidian": 2, - "book": 1 - } - }, - "agent_count": 2, - "target": "enchanting_table", - "number_of_target": 1, - "type": "techtree", - "max_depth": 0, - "depth": 0, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [ - "!getCraftingPlan" - ], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_jukebox_full_plan_requires_ctable__depth_1": { - "goal": "Collaborate with other agents to craft a jukebox", - "conversation": "Let's work together to craft a jukebox.", - "initial_inventory": { - "0": { - "diamond": 1 - }, - "1": { - "oak_log": 2, - "crafting_table": 1 - } - }, - "agent_count": 1, - "target": "jukebox", - "number_of_target": 1, - "type": "techtree", - "max_depth": 1, - "depth": 1, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_light_gray_wool_full_plan__depth_1": { - "goal": "Collaborate with other agents to craft light_gray_wool", - "conversation": "Let's work together to craft light_gray_wool.", - "initial_inventory": { - "0": { - "black_dye": 1 - }, - "1": { - "white_wool": 1, - "white_dye": 2 - } - }, - "agent_count": 1, - "target": "light_gray_wool", - "number_of_target": 1, - "type": "techtree", - "max_depth": 2, - "depth": 1, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [], - "1": [] - }, - "missing_items": [], - "requires_ctable": false - }, - "multiagent_crafting_blast_furnace_full_plan_requires_ctable__depth_1": { - "goal": "Collaborate with other agents to craft a blast_furnace", - "conversation": "Let's work together to craft a blast_furnace.", - "initial_inventory": { - "0": { - "iron_ingot": 5, - "smooth_stone": 3 - }, - "1": { - "cobblestone": 8, - "crafting_table": 1 - } - }, - "agent_count": 2, - "target": "blast_furnace", - "number_of_target": 1, - "type": "techtree", - "max_depth": 2, - "depth": 1, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_activator_rail_full_plan_requires_ctable__depth_2": { - "goal": "Collaborate with other agents to craft activator_rail", - "conversation": "Let's work together to craft activator_rail.", - "initial_inventory": { - "0": { - "iron_ingot": 3, - "oak_planks": 6 - }, - "1": { - "redstone": 1, - "iron_ingot": 3, - "crafting_table": 1 - } - }, - "agent_count": 1, - "target": "activator_rail", - "number_of_target": 1, - "type": "techtree", - "max_depth": 3, - "depth": 2, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_campfire_partial_plan_requires_ctable__depth_2": { - "goal": "Collaborate with other agents to craft campfire", - "conversation": "Let's work together to craft campfire.", - "initial_inventory": { - "0": { - "oak_log": 8 - }, - "1": { - "coal": 1, - "crafting_table": 1 - } - }, - "agent_count": 1, - "target": "campfire", - "number_of_target": 1, - "type": "techtree", - "max_depth": 3, - "depth": 2, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [ - "!getCraftingPlan" - ], - "1": [] - }, - "missing_items": [], - "requires_ctable": true - }, - "multiagent_crafting_crossbow_full_plan_requires_ctable__depth_2": { - "goal": "Collaborate with other agents to craft a crossbow", - "conversation": "Let's work together to craft a crossbow.", - "initial_inventory": { - "0": { - "oak_planks": 8, - "iron_ingot": 2 - }, - "1": { - "string": 2, - "crafting_table": 1 - } - }, - "agent_count": 1, - "target": "crossbow", - "number_of_target": 1, - "type": "techtree", - "max_depth": 3, - "depth": 2, - "timeout": 300, - "human_count": 1, - "usernames": [ - "izzycw" - ], - "blocked_actions": { - "0": [], - "1": [] - }, - "missing_items": [], - "requires_ctable": true } } \ No newline at end of file diff --git a/tasks/running_human_ai.md b/tasks/running_human_ai.md new file mode 100644 index 0000000..d1d04d6 --- /dev/null +++ b/tasks/running_human_ai.md @@ -0,0 +1,20 @@ +# Human AI Instructions + +## Construction +Press F3 to view the coordinates of the game. And pull up the file tasks/construction_tasks/church_blueprint.pdf +Run +``` +python tasks/evaluation_script.py --no_launch_world --template_profile profiles/tasks/construction_profile.json --task_path tasks/construction_tasks/human_ai/1_agent_1_human.json --usernames YOUR_USERNAME --num_agents 1 --insecure_coding +``` + +## Crafting + +``` +python tasks/evaluation_script.py --no_launch_world --template_profile profiles/tasks/crafting_profile.json --task_path tasks/crafting_tasks/human_ai/1_agent_1_human.json --usernames YOUR_USERNAME --num_agents 1 +``` + +## Cooking + +``` +python tasks/evaluation_script.py --no_launch_world --template_profile profiles/tasks/cooking_profile.json --task_path tasks/cooking_tasks/human_ai/1_agent_1_human.json --usernames YOUR_USERNAME --num_agents 1 +``` \ No newline at end of file From ef9fb747573946fb4ecd12b03baba98f38f69289 Mon Sep 17 00:00:00 2001 From: Isadora White Date: Mon, 26 May 2025 21:25:10 -0700 Subject: [PATCH 02/13] cleaning up human ai tasks --- .../human_ai/1_agent_1_human.json | 2 +- tasks/cooking_tasks/human_ai/1_agent_1_human.json | 8 ++++++-- .../1_agent_1_human.json | 2 +- .../1_agent_1_human_copy.json | 0 .../3_agent_1_human.json | 0 tasks/running_human_ai.md | 14 ++++++++++++++ 6 files changed, 22 insertions(+), 4 deletions(-) rename tasks/crafting_tasks/{human_ai_tasks => human_ai}/1_agent_1_human.json (96%) rename tasks/crafting_tasks/{human_ai_tasks => human_ai}/1_agent_1_human_copy.json (100%) rename tasks/crafting_tasks/{human_ai_tasks => human_ai}/3_agent_1_human.json (100%) diff --git a/tasks/construction_tasks/human_ai/1_agent_1_human.json b/tasks/construction_tasks/human_ai/1_agent_1_human.json index 2dfaa22..23171da 100644 --- a/tasks/construction_tasks/human_ai/1_agent_1_human.json +++ b/tasks/construction_tasks/human_ai/1_agent_1_human.json @@ -2351,7 +2351,7 @@ } }, "usernames": [ - "izzycw" + "erringnine" ] } } \ No newline at end of file diff --git a/tasks/cooking_tasks/human_ai/1_agent_1_human.json b/tasks/cooking_tasks/human_ai/1_agent_1_human.json index 0212f7a..a6b34b7 100644 --- a/tasks/cooking_tasks/human_ai/1_agent_1_human.json +++ b/tasks/cooking_tasks/human_ai/1_agent_1_human.json @@ -11,7 +11,8 @@ "Step 2: Go to the crafting table and surround the apple with the gold ingots to create a golden apple." ] }, - "agent_count": 2, + "agent_count": 1, + "human_count": 1, "target": { "bread": 1, "golden_apple": 1 @@ -29,6 +30,9 @@ "0": "Collaborate with other agents around you to make bread, golden_apple, The recipes are as follows:\nRecipe for bread:\n['Step 1: Go to the farm and collect 3 wheat.', 'Step 2: Go to the crafting table and use the wheat to craft bread.']\nRecipe for golden_apple:\n['Step 1: Get 1 apple and 8 gold ingots from your inventory or other bots.', 'Step 2: Go to the crafting table and surround the apple with the gold ingots to create a golden apple.']\n", "1": "Collaborate with other agents around you to make bread, golden_apple, The recipes are as follows:\nRecipe for bread:\n['Step 1: Go to the farm and collect 3 wheat.', 'Step 2: Go to the crafting table and use the wheat to craft bread.']\nRecipe for golden_apple:\n['Step 1: Get 1 apple and 8 gold ingots from your inventory or other bots.', 'Step 2: Go to the crafting table and surround the apple with the gold ingots to create a golden apple.']\n" }, - "conversation": "Let's collaborate to make bread, golden_apple, " + "conversation": "Let's collaborate to make bread, golden_apple, ", + "usernames": [ + "erringnine" + ] } } \ No newline at end of file diff --git a/tasks/crafting_tasks/human_ai_tasks/1_agent_1_human.json b/tasks/crafting_tasks/human_ai/1_agent_1_human.json similarity index 96% rename from tasks/crafting_tasks/human_ai_tasks/1_agent_1_human.json rename to tasks/crafting_tasks/human_ai/1_agent_1_human.json index cf01d07..f47a540 100644 --- a/tasks/crafting_tasks/human_ai_tasks/1_agent_1_human.json +++ b/tasks/crafting_tasks/human_ai/1_agent_1_human.json @@ -24,7 +24,7 @@ "missing_items": [], "human_count": 1, "usernames": [ - "izzycw" + "erringnine" ], "requires_ctable": false } diff --git a/tasks/crafting_tasks/human_ai_tasks/1_agent_1_human_copy.json b/tasks/crafting_tasks/human_ai/1_agent_1_human_copy.json similarity index 100% rename from tasks/crafting_tasks/human_ai_tasks/1_agent_1_human_copy.json rename to tasks/crafting_tasks/human_ai/1_agent_1_human_copy.json diff --git a/tasks/crafting_tasks/human_ai_tasks/3_agent_1_human.json b/tasks/crafting_tasks/human_ai/3_agent_1_human.json similarity index 100% rename from tasks/crafting_tasks/human_ai_tasks/3_agent_1_human.json rename to tasks/crafting_tasks/human_ai/3_agent_1_human.json diff --git a/tasks/running_human_ai.md b/tasks/running_human_ai.md index d1d04d6..1dcb8ec 100644 --- a/tasks/running_human_ai.md +++ b/tasks/running_human_ai.md @@ -1,5 +1,19 @@ # Human AI Instructions +## Finishing Installation + +Install the conda environment for running the experiments by executing this in your command line: + +``` +conda create --name mindcraft python=3.11 +conda activate mindcraft +pip install -r requirements.txt +``` + +## Setting up the world + +Setting up the world! Make sure your world has cheats enabled! You can do this on creation of your Minecraft world in the Minecraft console, or you can type ```/op @a``` in the chat or in the console of the world launched from the jar file. + ## Construction Press F3 to view the coordinates of the game. And pull up the file tasks/construction_tasks/church_blueprint.pdf Run From 133123983065e0af598f5dedb2065a439f6e79e9 Mon Sep 17 00:00:00 2001 From: Isadora White Date: Fri, 30 May 2025 17:39:47 -0500 Subject: [PATCH 03/13] small changes to tasks --- tasks/cooking_tasks/human_ai/1_agent_1_human.json | 2 +- tasks/crafting_tasks/human_ai/1_agent_1_human.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/cooking_tasks/human_ai/1_agent_1_human.json b/tasks/cooking_tasks/human_ai/1_agent_1_human.json index a6b34b7..20a4973 100644 --- a/tasks/cooking_tasks/human_ai/1_agent_1_human.json +++ b/tasks/cooking_tasks/human_ai/1_agent_1_human.json @@ -32,7 +32,7 @@ }, "conversation": "Let's collaborate to make bread, golden_apple, ", "usernames": [ - "erringnine" + "izzycw" ] } } \ No newline at end of file diff --git a/tasks/crafting_tasks/human_ai/1_agent_1_human.json b/tasks/crafting_tasks/human_ai/1_agent_1_human.json index f47a540..cf01d07 100644 --- a/tasks/crafting_tasks/human_ai/1_agent_1_human.json +++ b/tasks/crafting_tasks/human_ai/1_agent_1_human.json @@ -24,7 +24,7 @@ "missing_items": [], "human_count": 1, "usernames": [ - "erringnine" + "izzycw" ], "requires_ctable": false } From 0bffe111b12b6d249d9677386ebd1db1570552dc Mon Sep 17 00:00:00 2001 From: Isadora White Date: Sun, 1 Jun 2025 18:43:28 -0500 Subject: [PATCH 04/13] fixing weird conversation thing maybe --- src/agent/tasks/tasks.js | 33 +++++++++++++++++---------------- src/models/vllm.js | 1 + 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/agent/tasks/tasks.js b/src/agent/tasks/tasks.js index 1c00d95..f2d0fb2 100644 --- a/src/agent/tasks/tasks.js +++ b/src/agent/tasks/tasks.js @@ -423,10 +423,24 @@ export class Task { console.log(`Setting goal for agent ${this.agent.count_id}: ${agentGoal}`); } await executeCommand(this.agent, `!goal("${agentGoal}")`); + + if (this.data.conversation && this.agent.count_id === 0) { + let other_name = this.available_agents.filter(n => n !== this.name)[0]; + let waitCount = 0; + while (other_name === undefined && waitCount < 20) { + other_name = this.available_agents.filter(n => n !== this.name)[0]; + await new Promise((resolve) => setTimeout(resolve, 1000)); + waitCount++; + } + if (other_name === undefined && this.data.agent_count > 1) { + console.log('No other agents found. Task unsuccessful.'); + this.agent.killAll(); + } + await executeCommand(this.agent, `!startConversation("${other_name}", "${this.data.conversation}")`); + } } async initBotTask() { - await this.setAgentGoal(); await this.agent.bot.chat(`/clear ${this.name}`); console.log(`Cleared ${this.name}'s inventory.`); @@ -511,21 +525,8 @@ export class Task { this.agent.killAll(); } } - - if (this.data.conversation && this.agent.count_id === 0) { - let other_name = this.available_agents.filter(n => n !== this.name)[0]; - let waitCount = 0; - while (other_name === undefined && waitCount < 20) { - other_name = this.available_agents.filter(n => n !== this.name)[0]; - await new Promise((resolve) => setTimeout(resolve, 1000)); - waitCount++; - } - if (other_name === undefined && this.data.agent_count > 1) { - console.log('No other agents found. Task unsuccessful.'); - this.agent.killAll(); - } - await executeCommand(this.agent, `!startConversation("${other_name}", "${this.data.conversation}")`); - } + await new Promise((resolve) => setTimeout(resolve, 500)); + await this.setAgentGoal(); } diff --git a/src/models/vllm.js b/src/models/vllm.js index 52e3e5b..e9116ef 100644 --- a/src/models/vllm.js +++ b/src/models/vllm.js @@ -38,6 +38,7 @@ export class VLLM { try { console.log('Awaiting openai api response...') // console.log('Messages:', messages); + // todo set max_tokens, temperature, top_p, etc. in pack let completion = await this.vllm.chat.completions.create(pack); if (completion.choices[0].finish_reason == 'length') throw new Error('Context length exceeded'); From 0503ee34097d80d965ef23091c6143a48597f736 Mon Sep 17 00:00:00 2001 From: Isadora White Date: Sun, 1 Jun 2025 18:46:50 -0500 Subject: [PATCH 05/13] remove the conversation thingie from the set agent goal command --- src/agent/tasks/tasks.js | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/agent/tasks/tasks.js b/src/agent/tasks/tasks.js index f2d0fb2..a7948f5 100644 --- a/src/agent/tasks/tasks.js +++ b/src/agent/tasks/tasks.js @@ -423,21 +423,6 @@ export class Task { console.log(`Setting goal for agent ${this.agent.count_id}: ${agentGoal}`); } await executeCommand(this.agent, `!goal("${agentGoal}")`); - - if (this.data.conversation && this.agent.count_id === 0) { - let other_name = this.available_agents.filter(n => n !== this.name)[0]; - let waitCount = 0; - while (other_name === undefined && waitCount < 20) { - other_name = this.available_agents.filter(n => n !== this.name)[0]; - await new Promise((resolve) => setTimeout(resolve, 1000)); - waitCount++; - } - if (other_name === undefined && this.data.agent_count > 1) { - console.log('No other agents found. Task unsuccessful.'); - this.agent.killAll(); - } - await executeCommand(this.agent, `!startConversation("${other_name}", "${this.data.conversation}")`); - } } async initBotTask() { @@ -526,9 +511,21 @@ export class Task { } } await new Promise((resolve) => setTimeout(resolve, 500)); + if (this.data.conversation && this.agent.count_id === 0) { + let other_name = this.available_agents.filter(n => n !== this.name)[0]; + let waitCount = 0; + while (other_name === undefined && waitCount < 20) { + other_name = this.available_agents.filter(n => n !== this.name)[0]; + await new Promise((resolve) => setTimeout(resolve, 1000)); + waitCount++; + } + if (other_name === undefined && this.data.agent_count > 1) { + console.log('No other agents found. Task unsuccessful.'); + this.agent.killAll(); + } + await executeCommand(this.agent, `!startConversation("${other_name}", "${this.data.conversation}")`); + } await this.setAgentGoal(); - - } async teleportBots() { From 088b71a99aa8a7fd7b2258a2c9f97346ff1f4ca7 Mon Sep 17 00:00:00 2001 From: Isadora White Date: Mon, 9 Jun 2025 01:35:18 -0500 Subject: [PATCH 06/13] more friendly messages in the python evaluation script to make it more easy for the users to understand what is happening --- tasks/evaluation_script.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks/evaluation_script.py b/tasks/evaluation_script.py index cb380f5..4b5cb3c 100644 --- a/tasks/evaluation_script.py +++ b/tasks/evaluation_script.py @@ -179,7 +179,8 @@ def check_folder_results(folder_path): # Print summary print("\n=== Evaluation Results ===") - print(f"Total tasks evaluated: {results['total']}") + print("\nEvaluating Tasks!") + print(f"Results so far: {results['total']}") if "construction" not in folder_path: print(f"Successful tasks: {results['successful']}") @@ -517,7 +518,7 @@ def make_ops(agent_names, session_name): if agents_op: print("Agents are operators! You are good to go :D") else: - print("Agents are not operators! Something went wrong :(") + print("Agents are not operators! We will need to try making them operators again!") make_ops(agent_names, session_name) def check_agent_ops(agent_names, ops_file="ops.json"): From 6748b65fcbad0f3f5fcfe538977da6c7173ccff6 Mon Sep 17 00:00:00 2001 From: Isadora White <54700097+icwhite@users.noreply.github.com> Date: Mon, 9 Jun 2025 01:45:28 -0500 Subject: [PATCH 07/13] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 95f90ef..4c5b20d 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,15 @@ Do not connect this bot to public servers with coding enabled. This project allo ## Requirements -- [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) (up to v1.21.1, recommend v1.20.4) -- [Node.js Installed](https://nodejs.org/) (at least v14) +- [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) (up to v1.21.1, recommend v1.21.1) +- [Node.js Installed](https://nodejs.org/) (at least v14, recommended v20.9.0) - One of these: [OpenAI API Key](https://openai.com/blog/openai-api) | [Gemini API Key](https://aistudio.google.com/app/apikey) | [Anthropic API Key](https://docs.anthropic.com/claude/docs/getting-access-to-claude) | [Replicate API Key](https://replicate.com/) | [Hugging Face API Key](https://huggingface.co/) | [Groq API Key](https://console.groq.com/keys) | [Ollama Installed](https://ollama.com/download). | [Mistral API Key](https://docs.mistral.ai/getting-started/models/models_overview/) | [Qwen API Key [Intl.]](https://www.alibabacloud.com/help/en/model-studio/developer-reference/get-api-key)/[[cn]](https://help.aliyun.com/zh/model-studio/getting-started/first-api-call-to-qwen?) | [Novita AI API Key](https://novita.ai/settings?utm_source=github_mindcraft&utm_medium=github_readme&utm_campaign=link#key-management) | ## Install and Run 1. Make sure you have the requirements above. -2. Clone or download this repository (big green button) +2. Clone or download this repository (big green button) 'git clone https://github.com/kolbytn/mindcraft.git' 3. Rename `keys.example.json` to `keys.json` and fill in your API keys (you only need one). The desired model is set in `andy.json` or other profiles. For other models refer to the table below. @@ -28,7 +28,7 @@ Do not connect this bot to public servers with coding enabled. This project allo 6. Run `node main.js` from the installed directory -If you encounter issues, check the [FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) or find support on [discord](https://discord.gg/mp73p35dzC). We are currently not very responsive to github issues. +If you encounter issues, check the [FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) or find support on [discord](https://discord.gg/mp73p35dzC). We are currently not very responsive to github issues. To run tasks please refer to [Minecollab Instructions]((minecollab.md#installation) ## Tasks From 3a43b3c03c986d6c24b6465dc1e2625ec60702f7 Mon Sep 17 00:00:00 2001 From: Isadora White <54700097+icwhite@users.noreply.github.com> Date: Mon, 9 Jun 2025 01:46:08 -0500 Subject: [PATCH 08/13] Fix a small typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c5b20d..0835b4c 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Do not connect this bot to public servers with coding enabled. This project allo 6. Run `node main.js` from the installed directory -If you encounter issues, check the [FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) or find support on [discord](https://discord.gg/mp73p35dzC). We are currently not very responsive to github issues. To run tasks please refer to [Minecollab Instructions]((minecollab.md#installation) +If you encounter issues, check the [FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) or find support on [discord](https://discord.gg/mp73p35dzC). We are currently not very responsive to github issues. To run tasks please refer to [Minecollab Instructions](minecollab.md#installation) ## Tasks From 3661114321343df44e62aa4651ce3a3fd312b9bd Mon Sep 17 00:00:00 2001 From: Isadora White <54700097+icwhite@users.noreply.github.com> Date: Mon, 9 Jun 2025 01:55:47 -0500 Subject: [PATCH 09/13] Clarifying instructions for installing tmux --- minecollab.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/minecollab.md b/minecollab.md index 019d8b1..fa2fca3 100644 --- a/minecollab.md +++ b/minecollab.md @@ -55,9 +55,20 @@ pip install -r requirements.txt Then, you can run the evaluation_script **from the project root** using `python tasks/evaluation_script.py --task_path {your-task-path} --model {model you want to use}`. +### Tmux Installation +**MacOS**: +1. If brew isn't already installed run `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"` +2. `brew install tmux` +**Linux**: `apt-get -y install tmux` +**Windows**: You can not use tmux on Windows, but you can run tasks with the --no-launch-world flag. Run +``` +cd /tasks/server_data/ +java -jar server.jar +``` + If you want to run with vllm be sure to run with `--api vllm --url {your_url_for_vllm} --model {model_name}`, by default vllm will use http://127.0.0.1:8000/v1 as the url for quering the model! -When running with construction tasks, make sure to set the flag `--insecure_coding` so that the agents can be allowed to write freeform javascript code to complete the tasks. However, when using insecure coding it is highly recommended to use a docker container to avoid damage to your computer. +When running with construction tasks, make sure to set the flag `--insecure_coding` so that the agents can be allowed to write freeform javascript code to complete the tasks. However, when using insecure coding it is **highly recommended** to use a docker container to avoid damage to your computer. When running an experiment that requires more than 2 agents, use the `--num_agents` flag to match the number of agents in your task file. For example, if you are running a task file with 3 agents, use `--num_agents 3`. @@ -81,7 +92,7 @@ python tasks/evaluation_script.py --task_path {path_to_two_agent_construction_ta When you launch the evaluation script, you will see the minecraft server being launched. If you want to join this world, you can connect to it on the port localhost:55916 the way you would a standard Minecraft world (go to single player -> direct connection -> type in localhost:55916) It may take a few minutes for everything to be properly loaded - as first the agents need to be added to the world and given the correct permissions to use cheats and add inventory. After about 5 minutes everything should be loaded and working. If you wish to kill the experiment run `tmux kill-server`. Sometimes there will be issues copying the files, if this happens you can run the python file twice. -## Installation (without tmux) +## Windows Installation (without tmux) If you are on a machine that can't run tmux (like a Windows PC without WSL) or you don't care about doing evaluations only running tasks you can run the following script @@ -99,7 +110,7 @@ As you run, the evalaution script will evaluate the performance so far. It will ### Running multiple worlds in parallel -You can use `--num_parallel` to run multiple Minecraft worlds in parallel. This will launch `n` tmux shells, claled `server_i` and shell `i`, where `i` corresponds to ith parallel world. It will also copy worlds into `server_data_i` as well. On an M3 Mac with 34 GB of RAM, we can normally support up to 4 parallel worlds. When running an open source model, it is more likely you will be constrained by the throughput and size of your GPU RAM. On a cluster of 8 H100s you can expect to run 4 experiments in parallel. However, for best performance it is advisable to only use one parallel world. +You can use `--num_parallel` to run multiple Minecraft worlds in parallel. This will launch `n` tmux shells, called `server_i` and shell `i`, where `i` corresponds to ith parallel world. It will also copy worlds into `server_data_i` as well. On an M3 Mac with 34 GB of RAM, we can normally support up to 4 parallel worlds. When running an open source model, it is more likely you will be constrained by the throughput and size of your GPU RAM. On a cluster of 8 H100s you can expect to run 4 experiments in parallel. However, for best performance it is advisable to only use one parallel world. ### Using an S3 Bucket to store files To use S3 set the --s3 flag and the --bucket_name to use an s3 bucket to log all the files collected. It will also copy the /bots folder in this case with all of the files in there. From 00aa14ab5fbfcf2b53a6857dd76a4307b3e2f702 Mon Sep 17 00:00:00 2001 From: Isadora White <54700097+icwhite@users.noreply.github.com> Date: Mon, 9 Jun 2025 02:00:04 -0500 Subject: [PATCH 10/13] Update minecollab.md --- minecollab.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/minecollab.md b/minecollab.md index fa2fca3..5e3fb00 100644 --- a/minecollab.md +++ b/minecollab.md @@ -43,6 +43,12 @@ You can view the crafting task in action [here](https://www.youtube.com/shorts/V Please follow the installation docs in the README to install mindcraft. You can create a docker image using the Dockerfile. +If you don't own Minecraft you can run a limited version solely for offline games using these instructions: +1. Download the TLauncher https://tlauncher.org/en/ +2. Enter a username and select version 1.21.1 +3. Click "Multiplayer" and then "Direct Connection" +4. Then enter "localhost:55916" and hit `Join Server` + Download the relevant task files and server data files, you can find the link [here](https://drive.google.com/drive/folders/1XygbitBBTsNO6q_doEiZHmdETpnyRmCS). The tasks files are for specifying the tasks to run and the server data is for allowing the models to launch the task in the correct world automatically. **Unzip the server_data.zip in the base `tasks/` folder**. Then, set up your conda environment: From 5a403951d1fef71cf3109c554a22d81a05a719a8 Mon Sep 17 00:00:00 2001 From: Isadora White <54700097+icwhite@users.noreply.github.com> Date: Mon, 9 Jun 2025 02:00:36 -0500 Subject: [PATCH 11/13] Update minecollab.md --- minecollab.md | 1 + 1 file changed, 1 insertion(+) diff --git a/minecollab.md b/minecollab.md index 5e3fb00..945a4db 100644 --- a/minecollab.md +++ b/minecollab.md @@ -65,6 +65,7 @@ Then, you can run the evaluation_script **from the project root** using `python **MacOS**: 1. If brew isn't already installed run `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"` 2. `brew install tmux` + **Linux**: `apt-get -y install tmux` **Windows**: You can not use tmux on Windows, but you can run tasks with the --no-launch-world flag. Run ``` From 447b906ce314a7b89246df86c18686bc40c59245 Mon Sep 17 00:00:00 2001 From: Isadora White <54700097+icwhite@users.noreply.github.com> Date: Mon, 9 Jun 2025 02:00:57 -0500 Subject: [PATCH 12/13] Update minecollab.md --- minecollab.md | 1 + 1 file changed, 1 insertion(+) diff --git a/minecollab.md b/minecollab.md index 945a4db..29206d4 100644 --- a/minecollab.md +++ b/minecollab.md @@ -67,6 +67,7 @@ Then, you can run the evaluation_script **from the project root** using `python 2. `brew install tmux` **Linux**: `apt-get -y install tmux` + **Windows**: You can not use tmux on Windows, but you can run tasks with the --no-launch-world flag. Run ``` cd /tasks/server_data/ From a33465ce032cbd2d620a177e3e703a255b33db89 Mon Sep 17 00:00:00 2001 From: Max Robinson Date: Tue, 10 Jun 2025 13:56:56 -0500 Subject: [PATCH 13/13] better node version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0835b4c..07ce15a 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Do not connect this bot to public servers with coding enabled. This project allo ## Requirements - [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) (up to v1.21.1, recommend v1.21.1) -- [Node.js Installed](https://nodejs.org/) (at least v14, recommended v20.9.0) +- [Node.js Installed](https://nodejs.org/) (at least v18) - One of these: [OpenAI API Key](https://openai.com/blog/openai-api) | [Gemini API Key](https://aistudio.google.com/app/apikey) | [Anthropic API Key](https://docs.anthropic.com/claude/docs/getting-access-to-claude) | [Replicate API Key](https://replicate.com/) | [Hugging Face API Key](https://huggingface.co/) | [Groq API Key](https://console.groq.com/keys) | [Ollama Installed](https://ollama.com/download). | [Mistral API Key](https://docs.mistral.ai/getting-started/models/models_overview/) | [Qwen API Key [Intl.]](https://www.alibabacloud.com/help/en/model-studio/developer-reference/get-api-key)/[[cn]](https://help.aliyun.com/zh/model-studio/getting-started/first-api-call-to-qwen?) | [Novita AI API Key](https://novita.ai/settings?utm_source=github_mindcraft&utm_medium=github_readme&utm_campaign=link#key-management) | ## Install and Run