Merge branch 'main' into merge-main

This commit is contained in:
MaxRobinsonTheGreat 2025-04-19 13:43:11 -05:00
commit fff8d1a269
7 changed files with 4512 additions and 23 deletions

View file

@ -69,15 +69,16 @@ def analyze_json_file(file_path):
file_path (str): Path to the JSON file.
Returns:
str or None: The task outcome string if found, otherwise None.
bool: True if task was successful, False otherwise.
"""
try:
with open(file_path, 'r') as f:
data = json.load(f)
if 'turns' in data and isinstance(data['turns'], list):
for turn in reversed(data['turns']): # Check turns from the end
for turn in data['turns']: # Check all turns, not just from the end
if turn.get('role') == 'system' and isinstance(turn.get('content'), str):
if "Task successful ended with code : 2" in turn['content'] or "Task ended with score : 1" in turn["content"] or "Task ended in score: 1" in turn["content"]:
# print(f"Success found in {file_path}")
return True
return False
except FileNotFoundError:
@ -93,18 +94,17 @@ def analyze_json_file(file_path):
def extract_result(folder_path):
folder_name = os.path.basename(folder_path)
json_files = glob.glob(os.path.join(folder_path, "*.json"))
assert len(json_files) == 2, f"Expected 2 json files in {folder_name}, found {len(json_files)}"
if not json_files:
print(f"No JSON files found in {folder_name}")
return None
else:
outcome = False
# Check each JSON file in the folder for success indication
for json_file in json_files:
outcome = analyze_json_file(json_file)
if outcome:
if outcome: # If any file indicates success, return True
return True
return False
return False # Return False only if no files indicate success
def is_base(folder_path):
return "full_plan" in folder_path and "depth_0" in folder_path and "missing" not in folder_path
@ -307,7 +307,7 @@ if __name__ == "__main__":
folders = download_s3_folders(args.aws_bucket_name, args.s3_folder_prefix, args.local_download_dir)
else:
folders = get_immediate_subdirectories(args.local_download_dir)
print(folders)
# print(folders)
results = aggregate_results(folders)
print(results)

View file

@ -57,7 +57,6 @@ def analyze_json_file(file_path):
for turn in data["turns"]:
if turn.get("role") == "system" and "content" in turn:
if isinstance(turn["content"], str) and "Task ended with score : " in turn["content"]:
score_found = True
if "Task ended with score : 1" in turn["content"]:
return 1
elif "Task ended with score : 0" in turn["content"]:
@ -66,7 +65,8 @@ def analyze_json_file(file_path):
score = float(turn["content"].split(":")[-1].strip())
return score
return False
return None
except FileNotFoundError:
print(f"Error: File not found: {file_path}")
return None
@ -86,11 +86,14 @@ def extract_result(folder_path):
return None
else:
score = None
curr_score = 0
for json_file in json_files:
score = analyze_json_file(json_file)
if score is not None:
return score
return 0
max_score = max(score, curr_score)
curr_score = max_score
return curr_score
def aggregate_results(local_folders):
"""
@ -106,22 +109,92 @@ def aggregate_results(local_folders):
total = 0
successful = 0
successful_tasks = []
task_type = local_folders[0].split("/")[-2]
if "cooking" in task_type:
task_type = "cooking"
elif "techtree" in task_type:
task_type = "techtree"
elif "construction" in task_type:
task_type = "construction"
for folder_path in tqdm(local_folders):
folder_name = os.path.basename(folder_path)
try:
result = extract_result(folder_path)
if result == 1:
successful_tasks.append(folder_name)
if result is not None:
total += 1
successful += result
except Exception as e:
print(f"Error processing {folder_name}: {e}")
successful_tasks.sort()
if task_type == "construction":
successful = successful / total
return {
"total": total,
"successful": successful,
}
def check_folder_results(folder_path):
"""
Evaluate all JSON files in a folder and its subfolders and calculate success metrics.
Args:
folder_path (str): Path to the folder containing JSON log files.
Returns:
dict: A dictionary with success metrics.
"""
print(f"Checking results in folder: {folder_path}")
# Check if the folder exists
if not os.path.exists(folder_path):
print(f"Error: Folder not found: {folder_path}")
return None
# Find all subfolders (task IDs) in the given folder
if os.path.isdir(folder_path):
subfolders = [f for f in glob.glob(os.path.join(folder_path, "*")) if os.path.isdir(f)]
if subfolders:
# If there are subfolders, evaluate each subfolder
print(f"Found {len(subfolders)} subfolders to evaluate")
results = aggregate_results(subfolders)
else:
# If no subfolders, treat the folder itself as a results folder
print("No subfolders found, evaluating the folder itself")
results = aggregate_results([folder_path])
# Calculate success rate
if results["total"] > 0:
results["success_rate"] = results["successful"] / results["total"]
else:
results["success_rate"] = 0.0
# Print summary
print("\n=== Evaluation Results ===")
print(f"Total tasks evaluated: {results['total']}")
if "construction" not in folder_path:
print(f"Successful tasks: {results['successful']}")
if "construction" not in folder_path:
print(f"Success rate: {results['success_rate']:.2f}")
else:
print(f"Success rate: {results['successful']:.2f}")
return results
else:
print(f"Error: {folder_path} is not a directory")
return None
def read_settings(file_path):
"""Read and parse the settings.js file to get agent profiles."""
with open(file_path, 'r', encoding='utf-8') as file:
@ -722,9 +795,16 @@ def main():
parser.add_argument('--num_examples', default=2, type=int, help='Maximum number of turns before summarizing')
parser.add_argument('--no-pruning', action='store_true', help='Disable pruning of the actions')
parser.add_argument('--block_conversation', action='store_true', help='Block conversation actions')
parser.add_argument('--check', metavar='FOLDER_PATH', help='Check and evaluate results in the specified folder without running experiments')
args = parser.parse_args()
print(args)
# If --check flag is provided, evaluate results in the specified folder and exit
if args.check:
check_folder_results(args.check)
return
if not args.no_launch_world:
try:
subprocess.run(['tmux', 'kill-server'], check=True)

View file

@ -1054,6 +1054,7 @@ export function blueprintToTask(blueprint_data, num_agents) {
}
let give_agent = 0;
console.log("materials", blueprint_data.materials)
for (const key of Object.keys(blueprint_data.materials)) {
initialInventory[JSON.stringify(give_agent)][key] = blueprint_data.materials[key];
give_agent = (give_agent + 1) % num_agents;
@ -1063,7 +1064,7 @@ export function blueprintToTask(blueprint_data, num_agents) {
type: "construction",
goal: "Make a structure with the blueprint below",
conversation: "Let's share materials and make a structure with the blueprint",
agent_count: 2,
agent_count: num_agents,
blueprint: blueprint_data,
initial_inventory: initialInventory,
};

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,699 @@
{
"pyramid_three_agents": {
"type": "construction",
"goal": "Make a structure with the blueprint below",
"conversation": "Let's share materials and make a structure with the blueprint",
"agent_count": 3,
"blueprint": {
"materials": {
"polished_granite": 1,
"gold_block": 27,
"stone_bricks": 41,
"polished_andesite": 34,
"quartz_block": 16,
"stone": 23,
"polished_diorite": 21,
"quartz_pillar": 2,
"glowstone": 3
},
"levels": [
{
"level": 0,
"coordinates": [
-60,
-60,
6
],
"placement": [
[
"polished_granite",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"gold_block",
"stone_bricks",
"polished_andesite",
"gold_block",
"quartz_block",
"polished_andesite",
"gold_block",
"stone_bricks",
"gold_block"
],
[
"air",
"stone_bricks",
"polished_andesite",
"stone",
"polished_diorite",
"polished_andesite",
"stone",
"stone_bricks",
"polished_diorite",
"gold_block"
],
[
"air",
"polished_andesite",
"stone",
"polished_diorite",
"polished_andesite",
"stone",
"stone_bricks",
"polished_diorite",
"stone",
"stone_bricks"
],
[
"air",
"gold_block",
"polished_diorite",
"polished_andesite",
"stone",
"stone_bricks",
"polished_diorite",
"stone",
"stone_bricks",
"polished_andesite"
],
[
"air",
"quartz_block",
"polished_andesite",
"stone",
"stone_bricks",
"polished_diorite",
"stone",
"stone_bricks",
"polished_andesite",
"quartz_block"
],
[
"air",
"polished_andesite",
"stone",
"stone_bricks",
"polished_diorite",
"stone",
"stone_bricks",
"polished_andesite",
"polished_diorite",
"stone_bricks"
],
[
"air",
"gold_block",
"stone_bricks",
"polished_diorite",
"stone",
"stone_bricks",
"polished_andesite",
"polished_diorite",
"stone_bricks",
"polished_andesite"
],
[
"air",
"stone_bricks",
"polished_diorite",
"stone",
"stone_bricks",
"polished_andesite",
"polished_diorite",
"stone_bricks",
"polished_andesite",
"gold_block"
],
[
"air",
"gold_block",
"gold_block",
"stone_bricks",
"polished_andesite",
"quartz_block",
"stone_bricks",
"polished_andesite",
"gold_block",
"gold_block"
]
]
},
{
"level": 1,
"coordinates": [
-60,
-59,
6
],
"placement": [
[
"quartz_pillar",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"gold_block",
"stone_bricks",
"polished_andesite",
"quartz_block",
"stone_bricks",
"polished_andesite",
"gold_block",
"air"
],
[
"air",
"air",
"stone_bricks",
"stone",
"polished_diorite",
"polished_andesite",
"stone",
"stone_bricks",
"stone_bricks",
"air"
],
[
"air",
"air",
"polished_andesite",
"polished_diorite",
"polished_andesite",
"stone",
"stone_bricks",
"polished_diorite",
"polished_andesite",
"air"
],
[
"air",
"air",
"quartz_block",
"polished_andesite",
"stone",
"glowstone",
"polished_diorite",
"stone",
"quartz_block",
"air"
],
[
"air",
"air",
"stone_bricks",
"stone",
"stone_bricks",
"polished_diorite",
"stone",
"stone_bricks",
"stone_bricks",
"air"
],
[
"air",
"air",
"polished_andesite",
"stone_bricks",
"polished_diorite",
"stone",
"stone_bricks",
"polished_andesite",
"polished_andesite",
"air"
],
[
"air",
"air",
"gold_block",
"stone_bricks",
"polished_andesite",
"quartz_block",
"stone_bricks",
"polished_andesite",
"gold_block",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
]
]
},
{
"level": 2,
"coordinates": [
-60,
-58,
6
],
"placement": [
[
"quartz_pillar",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"gold_block",
"stone_bricks",
"quartz_block",
"gold_block",
"gold_block",
"air",
"air"
],
[
"air",
"air",
"air",
"stone_bricks",
"polished_diorite",
"polished_andesite",
"stone",
"polished_andesite",
"air",
"air"
],
[
"air",
"air",
"air",
"quartz_block",
"polished_andesite",
"glowstone",
"stone_bricks",
"quartz_block",
"air",
"air"
],
[
"air",
"air",
"air",
"gold_block",
"stone",
"stone_bricks",
"polished_diorite",
"stone_bricks",
"air",
"air"
],
[
"air",
"air",
"air",
"gold_block",
"polished_andesite",
"quartz_block",
"stone_bricks",
"gold_block",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
]
]
},
{
"level": 3,
"coordinates": [
-60,
-57,
6
],
"placement": [
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"gold_block",
"quartz_block",
"gold_block",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"quartz_block",
"glowstone",
"quartz_block",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"gold_block",
"quartz_block",
"gold_block",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
]
]
},
{
"level": 4,
"coordinates": [
-60,
-56,
6
],
"placement": [
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"gold_block",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
],
[
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air",
"air"
]
]
}
]
},
"initial_inventory": {
"0": {
"diamond_pickaxe": 1,
"diamond_axe": 1,
"diamond_shovel": 1,
"polished_granite": 1,
"polished_andesite": 34,
"polished_diorite": 21
},
"1": {
"diamond_pickaxe": 1,
"diamond_axe": 1,
"diamond_shovel": 1,
"gold_block": 27,
"quartz_block": 16,
"quartz_pillar": 2
},
"2": {
"diamond_pickaxe": 1,
"diamond_axe": 1,
"diamond_shovel": 1,
"stone_bricks": 41,
"stone": 23,
"glowstone": 3
}
}
}
}

View file

@ -14,32 +14,33 @@ bot.on('spawn', async () => {
console.log("Bot spawned. Starting blueprint check...");
// set this to be minX, minY, minZ
const startCoord = {
x: -60,
x: -124,
y: 1,
z: 6,
z: 133,
}
bot.chat(`/tp andy ${startCoord.x} ${startCoord.y} ${startCoord.z}`);
const yOffset = 5;
const xOffset = 10;
const zOffset = 10;
const yOffset = 2;
const xOffset = 30;
const zOffset = 20;
const taskFilePath = '/Users/isadorawhite/izzy_mindcraft/mindcraft/tasks/construction_tasks/custom/pyramid.json';
const task_name = "pyramid";
const taskFilePath = '/Users/isadorawhite/izzy_mindcraft/mindcraft/tasks/construction_tasks/custom/flower_three_agents.json';
const task_name = "flower_three_agents";
setTimeout(async () => {
let task_blueprint = await worldToBlueprint(startCoord, yOffset, xOffset, zOffset, bot);
for (const level of task_blueprint.levels) {
for (let i = 0; i < task_blueprint.levels.length; i++) {
// Perform operations on each level
const level = task_blueprint.levels[i];
console.log("Level coordinates:", level.coordinates);
const new_coordinates = [level.coordinates[0], -60, level.coordinates[2]];
const new_coordinates = [level.coordinates[0], -60 + i, level.coordinates[2]];
level.coordinates = new_coordinates;
console.log("New coordinates:", level.coordinates);
}
console.log("Blueprint generated:", task_blueprint.levels[0].coordinates);
const task = blueprintToTask(task_blueprint, 2);
const task = blueprintToTask(task_blueprint, 3);
const task_collection = {}
task_collection[task_name] = task;