mirror of
https://github.com/kolbytn/mindcraft.git
synced 2025-07-25 17:35:25 +02:00
Merge branch 'cooking_tasks' of https://github.com/icwhite/mindcraft into cooking_tasks
This commit is contained in:
Ayush Maniar
commit
0c237b76da
25 changed files with 622 additions and 193 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -13,3 +13,6 @@ services/viaproxy/plugins/**
|
|||
services/viaproxy/ViaLoader/**
|
||||
services/viaproxy/saves.json
|
||||
services/viaproxy/viaproxy.yml
|
||||
tmp/
|
||||
wandb/
|
||||
experiments/
|
||||
69
README.md
69
README.md
|
|
@ -2,11 +2,11 @@
|
|||
|
||||
Crafting minds for Minecraft with LLMs and [Mineflayer!](https://prismarinejs.github.io/mineflayer/#/)
|
||||
|
||||
[FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) | [Discord Support](https://discord.gg/mp73p35dzC) | [Blog Post](https://kolbynottingham.com/mindcraft/) | [Contributor TODO](https://github.com/users/kolbytn/projects/1)
|
||||
[FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) | [Discord Support](https://discord.gg/mp73p35dzC) | [Video Tutorial](https://www.youtube.com/watch?v=gRotoL8P8D8) | [Blog Post](https://kolbynottingham.com/mindcraft/) | [Contributor TODO](https://github.com/users/kolbytn/projects/1)
|
||||
|
||||
|
||||
> [!WARNING]
|
||||
Do not connect this bot to public servers with coding enabled. This project allows an LLM to write/execute code on your computer. While the code is sandboxed, it is still vulnerable to injection attacks on public servers. Code writing is disabled by default, you can enable it by setting `allow_insecure_coding` to `true` in `settings.js`. We strongly recommend running with additional layers of security such as docker containers. Ye be warned.
|
||||
> [!Caution]
|
||||
Do not connect this bot to public servers with coding enabled. This project allows an LLM to write/execute code on your computer. The code is sandboxed, but still vulnerable to injection attacks. Code writing is disabled by default, you can enable it by setting `allow_insecure_coding` to `true` in `settings.js`. Ye be warned.
|
||||
|
||||
## Requirements
|
||||
|
||||
|
|
@ -30,30 +30,31 @@ Do not connect this bot to public servers with coding enabled. This project allo
|
|||
|
||||
If you encounter issues, check the [FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) or find support on [discord](https://discord.gg/mp73p35dzC). We are currently not very responsive to github issues.
|
||||
|
||||
## Customization
|
||||
## Model Customization
|
||||
|
||||
You can configure project details in `settings.js`. [See file.](settings.js)
|
||||
|
||||
You can configure the agent's name, model, and prompts in their profile like `andy.json`.
|
||||
You can configure the agent's name, model, and prompts in their profile like `andy.json` with the `model` field. For comprehensive details, see [Model Specifications](#model-specifications).
|
||||
|
||||
| API | Config Variable | Example Model name | Docs |
|
||||
|------|------|------|------|
|
||||
| OpenAI | `OPENAI_API_KEY` | `gpt-4o-mini` | [docs](https://platform.openai.com/docs/models) |
|
||||
| Google | `GEMINI_API_KEY` | `gemini-pro` | [docs](https://ai.google.dev/gemini-api/docs/models/gemini) |
|
||||
| Anthropic | `ANTHROPIC_API_KEY` | `claude-3-haiku-20240307` | [docs](https://docs.anthropic.com/claude/docs/models-overview) |
|
||||
| Replicate | `REPLICATE_API_KEY` | `replicate/meta/meta-llama-3-70b-instruct` | [docs](https://replicate.com/collections/language-models) |
|
||||
| Ollama (local) | n/a | `llama3` | [docs](https://ollama.com/library) |
|
||||
| Groq | `GROQCLOUD_API_KEY` | `groq/mixtral-8x7b-32768` | [docs](https://console.groq.com/docs/models) |
|
||||
| Hugging Face | `HUGGINGFACE_API_KEY` | `huggingface/mistralai/Mistral-Nemo-Instruct-2407` | [docs](https://huggingface.co/models) |
|
||||
| Novita AI | `NOVITA_API_KEY` | `gryphe/mythomax-l2-13b` | [docs](https://novita.ai/model-api/product/llm-api?utm_source=github_mindcraft&utm_medium=github_readme&utm_campaign=link) |
|
||||
| Qwen | `QWEN_API_KEY` | `qwen-max` | [Intl.](https://www.alibabacloud.com/help/en/model-studio/developer-reference/use-qwen-by-calling-api)/[cn](https://help.aliyun.com/zh/model-studio/getting-started/models) |
|
||||
| Mistral | `MISTRAL_API_KEY` | `mistral-large-latest` | [docs](https://docs.mistral.ai/getting-started/models/models_overview/) |
|
||||
| xAI | `XAI_API_KEY` | `grok-beta` | [docs](https://docs.x.ai/docs) |
|
||||
| `openai` | `OPENAI_API_KEY` | `gpt-4o-mini` | [docs](https://platform.openai.com/docs/models) |
|
||||
| `google` | `GEMINI_API_KEY` | `gemini-pro` | [docs](https://ai.google.dev/gemini-api/docs/models/gemini) |
|
||||
| `anthropic` | `ANTHROPIC_API_KEY` | `claude-3-haiku-20240307` | [docs](https://docs.anthropic.com/claude/docs/models-overview) |
|
||||
| `replicate` | `REPLICATE_API_KEY` | `replicate/meta/meta-llama-3-70b-instruct` | [docs](https://replicate.com/collections/language-models) |
|
||||
| `ollama` (local) | n/a | `llama3` | [docs](https://ollama.com/library) |
|
||||
| `groq` | `GROQCLOUD_API_KEY` | `groq/mixtral-8x7b-32768` | [docs](https://console.groq.com/docs/models) |
|
||||
| `huggingface` | `HUGGINGFACE_API_KEY` | `huggingface/mistralai/Mistral-Nemo-Instruct-2407` | [docs](https://huggingface.co/models) |
|
||||
| `novita` | `NOVITA_API_KEY` | `gryphe/mythomax-l2-13b` | [docs](https://novita.ai/model-api/product/llm-api?utm_source=github_mindcraft&utm_medium=github_readme&utm_campaign=link) |
|
||||
| `qwen` | `QWEN_API_KEY` | `qwen-max` | [Intl.](https://www.alibabacloud.com/help/en/model-studio/developer-reference/use-qwen-by-calling-api)/[cn](https://help.aliyun.com/zh/model-studio/getting-started/models) |
|
||||
| `xai` | `MISTRAL_API_KEY` | `mistral-large-latest` | [docs](https://docs.mistral.ai/getting-started/models/models_overview/) |
|
||||
| `deepseek` | `XAI_API_KEY` | `grok-beta` | [docs](https://docs.x.ai/docs) |
|
||||
| `openrouter` | `OPENROUTER_API_KEY` | `openrouter/anthropic/claude-3.5-sonnet` | [docs](https://openrouter.ai/models) |
|
||||
|
||||
If you use Ollama, to install the models used by default (generation and embedding), execute the following terminal command:
|
||||
`ollama pull llama3 && ollama pull nomic-embed-text`
|
||||
|
||||
## Online Servers
|
||||
### Online Servers
|
||||
To connect to online servers your bot will need an official Microsoft/Minecraft account. You can use your own personal one, but will need another account if you want to connect too and play with it. To connect, change these lines in `settings.js`:
|
||||
```javascript
|
||||
"host": "111.222.333.444",
|
||||
|
|
@ -62,7 +63,7 @@ To connect to online servers your bot will need an official Microsoft/Minecraft
|
|||
|
||||
// rest is same...
|
||||
```
|
||||
> [!CAUTION]
|
||||
> [!Important]
|
||||
> The bot's name in the profile.json must exactly match the Minecraft profile name! Otherwise the bot will spam talk to itself.
|
||||
|
||||
To use different accounts, Mindcraft will connect with the account that the Minecraft launcher is currently using. You can switch accounts in the launcer, then run `node main.js`, then switch to your main account after the bot has connected.
|
||||
|
|
@ -87,25 +88,17 @@ When running in docker, if you want the bot to join your local minecraft server,
|
|||
|
||||
To connect to an unsupported minecraft version, you can try to use [viaproxy](services/viaproxy/README.md)
|
||||
|
||||
## Bot Profiles
|
||||
# Bot Profiles
|
||||
|
||||
Bot profiles are json files (such as `andy.json`) that define:
|
||||
|
||||
1. Bot backend LLMs to use for chat and embeddings.
|
||||
1. Bot backend LLMs to use for talking, coding, and embedding.
|
||||
2. Prompts used to influence the bot's behavior.
|
||||
3. Examples help the bot perform tasks.
|
||||
|
||||
### Specifying Profiles via Command Line
|
||||
## Model Specifications
|
||||
|
||||
By default, the program will use the profiles specified in `settings.js`. You can specify one or more agent profiles using the `--profiles` argument:
|
||||
|
||||
```bash
|
||||
node main.js --profiles ./profiles/andy.json ./profiles/jill.json
|
||||
```
|
||||
|
||||
### Model Specifications
|
||||
|
||||
LLM models can be specified as simply as `"model": "gpt-4o"`. However, you can specify different models for chat, coding, and embeddings.
|
||||
LLM models can be specified simply as `"model": "gpt-4o"`. However, you can use different models for chat, coding, and embeddings.
|
||||
You can pass a string or an object for these fields. A model object must specify an `api`, and optionally a `model`, `url`, and additional `params`.
|
||||
|
||||
```json
|
||||
|
|
@ -131,11 +124,21 @@ You can pass a string or an object for these fields. A model object must specify
|
|||
|
||||
```
|
||||
|
||||
`model` is used for chat, `code_model` is used for newAction coding, and `embedding` is used to embed text for example selection. If `code_model` is not specified, then it will use `model` for coding.
|
||||
`model` is used for chat, `code_model` is used for newAction coding, and `embedding` is used to embed text for example selection. If `code_model` or `embedding` are not specified, they will use `model` by default. Not all APIs have an embedding model.
|
||||
|
||||
All apis have default models and urls, so those fields are optional. Note some apis have no embedding model, so they will default to word overlap to retrieve examples.
|
||||
All apis have default models and urls, so those fields are optional. The `params` field is optional and can be used to specify additional parameters for the model. It accepts any key-value pairs supported by the api. Is not supported for embedding models.
|
||||
|
||||
The `params` field is optional and can be used to specify additional parameters for the model. It accepts any key-value pairs supported by the api. Is not supported for embedding models.
|
||||
## Embedding Models
|
||||
|
||||
Embedding models are used to embed and efficiently select relevant examples for conversation and coding.
|
||||
|
||||
Supported Embedding APIs: `openai`, `google`, `replicate`, `huggingface`, `novita`
|
||||
|
||||
If you try to use an unsupported model, then it will default to a simple word-overlap method. Expect reduced performance, recommend mixing APIs to ensure embedding support.
|
||||
|
||||
## Specifying Profiles via Command Line
|
||||
|
||||
By default, the program will use the profiles specified in `settings.js`. You can specify one or more agent profiles using the `--profiles` argument: `node main.js --profiles ./profiles/andy.json ./profiles/jill.json`
|
||||
|
||||
## Patches
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,13 @@
|
|||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
|
||||
def read_settings(file_path):
|
||||
"""Read and parse the settings.js file to get agent profiles."""
|
||||
|
|
@ -30,7 +34,7 @@ def read_settings(file_path):
|
|||
## profiles is a list of strings like "./andy.json" and "./bob.json"
|
||||
|
||||
agent_names = [profile.split('/')[-1].split('.')[0] for profile in profiles]
|
||||
return agent_names
|
||||
return agent_names
|
||||
|
||||
def check_task_completion(agents):
|
||||
"""Check memory.json files of all agents to determine task success/failure."""
|
||||
|
|
@ -80,68 +84,267 @@ def update_results_file(task_id, success_count, total_count, time_taken, experim
|
|||
f.write(f"Average time per experiment: {total_time / total_count:.2f} seconds\n")
|
||||
f.write(f"Last updated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
|
||||
|
||||
def run_experiment(task_path, task_id, num_exp):
|
||||
"""Run the specified number of experiments and track results."""
|
||||
# Read agent profiles from settings.js
|
||||
agents = read_settings(file_path="settings.js")
|
||||
print(f"Detected agents: {agents}")
|
||||
|
||||
def set_environment_variable_tmux_session(session_name, key, value):
|
||||
"""Set an environment variable for the current process."""
|
||||
subprocess.run(["tmux", "send-keys", "-t", session_name, f"export {key}={value}", "C-m"])
|
||||
|
||||
def launch_parallel_experiments(task_path,
|
||||
num_exp,
|
||||
exp_name,
|
||||
num_agents=2,
|
||||
model="gpt-4o",
|
||||
num_parallel=1):
|
||||
|
||||
# Generate timestamp at the start of experiments
|
||||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
results_filename = f"results_{task_id}_{timestamp}.txt"
|
||||
print(f"Results will be saved to: {results_filename}")
|
||||
with open(task_path, 'r', encoding='utf-8') as file:
|
||||
content = file.read()
|
||||
json_data = json.loads(content)
|
||||
|
||||
task_ids = json_data.keys()
|
||||
|
||||
# split the task_ids into num_parallel groups
|
||||
task_ids = list(task_ids)
|
||||
task_ids_split = [task_ids[i::num_parallel] for i in range(num_parallel)]
|
||||
|
||||
servers = create_server_files("../server_data/", num_parallel)
|
||||
date_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
experiments_folder = f"experiments/{exp_name}_{date_time}"
|
||||
exp_name = f"{exp_name}_{date_time}"
|
||||
|
||||
# start wandb
|
||||
os.makedirs(experiments_folder, exist_ok=True)
|
||||
for i, server in enumerate(servers):
|
||||
launch_server_experiment(task_path, task_ids_split[i], num_exp, server, experiments_folder, exp_name)
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
def launch_server_experiment(task_path,
|
||||
task_ids,
|
||||
num_exp,
|
||||
server,
|
||||
experiments_folder,
|
||||
exp_name="exp",
|
||||
num_agents=2,
|
||||
model="gpt-4o"):
|
||||
"""
|
||||
Launch a Minecraft server and run experiments on it.
|
||||
@param task_path: Path to the task file
|
||||
@param task_ids: IDs of the tasks to run
|
||||
@param num_exp: Number of experiments to run
|
||||
@param server: Tuple containing server path and port
|
||||
@param experiments_folder: Folder to store experiment results
|
||||
@param exp_name: Name of the experiment for wandb dataset
|
||||
@param num_agents: Number of agents to run
|
||||
@param model: Model to use for the agents
|
||||
"""
|
||||
server_path, server_port = server
|
||||
edit_file(os.path.join(server_path, "server.properties"), {"server-port": server_port})
|
||||
mindserver_port = server_port - 55916 + 8080
|
||||
|
||||
success_count = 0
|
||||
experiment_results = []
|
||||
|
||||
for exp_num in range(num_exp):
|
||||
print(f"\nRunning experiment {exp_num + 1}/{num_exp}")
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
# Run the node command
|
||||
# set up server and agents
|
||||
session_name = str(server_port - 55916)
|
||||
if num_agents == 2:
|
||||
agent_names = [f"andy_{session_name}", f"jill_{session_name}"]
|
||||
models = [model] * 2
|
||||
else:
|
||||
agent_names = [f"andy_{session_name}", f"jill_{session_name}", f"bob_{session_name}"]
|
||||
models = [model] * 3
|
||||
make_profiles(agent_names, models)
|
||||
|
||||
# edit_file("settings.js", {"profiles": [f"./{agent}.json" for agent in agent_names]})
|
||||
agent_profiles = [f"./{agent}.json" for agent in agent_names]
|
||||
agent_profiles_str = f"\'[\"{agent_profiles[0]}\", \"{agent_profiles[1]}\"]\'"
|
||||
print(agent_profiles_str)
|
||||
launch_world(server_path, session_name="server_" + session_name, agent_names=agent_names)
|
||||
|
||||
subprocess.run(['tmux', 'new-session', '-d', '-s', session_name], check=True)
|
||||
|
||||
# set environment variables
|
||||
set_environment_variable_tmux_session(session_name, "MINECRAFT_PORT", server_port)
|
||||
set_environment_variable_tmux_session(session_name, "MINDSERVER_PORT", mindserver_port)
|
||||
set_environment_variable_tmux_session(session_name, "PROFILES", agent_profiles_str)
|
||||
|
||||
script_content = ""
|
||||
for task_id in task_ids:
|
||||
cmd = f"node main.js --task_path {task_path} --task_id {task_id}"
|
||||
try:
|
||||
subprocess.run(cmd, shell=True, check=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"Error running experiment: {e}")
|
||||
continue
|
||||
|
||||
# Check if task was successful
|
||||
success = check_task_completion(agents)
|
||||
if success:
|
||||
success_count += 1
|
||||
print(f"Experiment {exp_num + 1} successful")
|
||||
else:
|
||||
print(f"Experiment {exp_num + 1} failed")
|
||||
|
||||
end_time = time.time()
|
||||
time_taken = end_time - start_time
|
||||
|
||||
# Store individual experiment result
|
||||
experiment_results.append({
|
||||
'success': success,
|
||||
'time_taken': time_taken
|
||||
})
|
||||
|
||||
# Update results file after each experiment using the constant filename
|
||||
update_results_file(task_id, success_count, exp_num + 1, time_taken, experiment_results, results_filename)
|
||||
|
||||
# Small delay between experiments
|
||||
time.sleep(1)
|
||||
|
||||
final_ratio = success_count / num_exp
|
||||
print(f"\nExperiments completed. Final success ratio: {final_ratio:.2f}")
|
||||
cp_cmd = f"cp {agent_names[0]}.json {server_path}bots/{agent_names[0]}/profile.json"
|
||||
for _ in range(num_exp):
|
||||
script_content += f"{cmd}\n"
|
||||
script_content += "sleep 2\n"
|
||||
for agent in agent_names:
|
||||
cp_cmd = f"cp bots/{agent}/memory.json {experiments_folder}/{task_id}_{agent}_{_}.json"
|
||||
script_content += f"{cp_cmd}\n"
|
||||
script_content += "sleep 1\n"
|
||||
script_content += f"echo 'Uploading {experiments_folder}/{task_id}_{agent}_{_}.json to wandb'\n"
|
||||
wandb_cmd = f"wandb artifact put {experiments_folder}/{task_id}_{agent}_{_}.json --name {exp_name}_{task_id}_{agent}_{_} --type dataset"
|
||||
script_content += f"echo '{wandb_cmd}'\n"
|
||||
script_content += f"{wandb_cmd}\n"
|
||||
script_content += "sleep 1\n"
|
||||
script_content += "sleep 1\n"
|
||||
|
||||
# Create a temporary shell script file
|
||||
script_file = f"./tmp/experiment_script_{session_name}.sh"
|
||||
|
||||
script_dir = os.path.dirname(script_file)
|
||||
os.makedirs(script_dir, exist_ok=True)
|
||||
|
||||
# Call the function before writing the script file
|
||||
with open(script_file, 'w') as f:
|
||||
f.write(script_content)
|
||||
|
||||
script_file_run = "bash " + script_file
|
||||
|
||||
# Execute the shell script using subprocess
|
||||
subprocess.run(["tmux", "send-keys", "-t", session_name, script_file_run, "C-m"])
|
||||
|
||||
|
||||
# subprocess.run(["tmux", "send-keys", "-t", session_name, f"/op {agent_names[0]}", "C-m"])
|
||||
|
||||
def make_profiles(agent_names, models):
|
||||
assert len(agent_names) == len(models)
|
||||
for index in range(len(agent_names)):
|
||||
content = {"name": agent_names[index], "model": models[index], "modes": {"hunting": False}}
|
||||
with open(f"{agent_names[index]}.json", 'w') as f:
|
||||
json.dump(content, f)
|
||||
|
||||
def create_server_files(source_path, num_copies):
|
||||
"""Create multiple copies of server files for parallel experiments."""
|
||||
print("Creating server files...")
|
||||
print(num_copies)
|
||||
servers = []
|
||||
for i in range(num_copies):
|
||||
dest_path = f"../server_data_{i}/"
|
||||
copy_server_files(source_path, dest_path)
|
||||
print(dest_path)
|
||||
edit_file(dest_path + "server.properties", {"server-port": 55916 + i})
|
||||
# edit_server_properties_file(dest_path, 55916 + i)
|
||||
servers.append((dest_path, 55916 + i))
|
||||
return servers
|
||||
|
||||
def edit_file(file, content_dict):
|
||||
try:
|
||||
with open(file, 'r') as f:
|
||||
lines = f.readlines()
|
||||
with open(file, 'w') as f:
|
||||
for line in lines:
|
||||
for key, value in content_dict.items():
|
||||
if line.startswith(key):
|
||||
f.write(f"{key}={value}\n")
|
||||
else:
|
||||
f.write(line)
|
||||
print(f"{file} updated with {content_dict}")
|
||||
except Exception as e:
|
||||
print(f"Error editing file {file}: {e}")
|
||||
|
||||
def clean_up_server_files(num_copies):
|
||||
"""Delete server files from multiple locations."""
|
||||
for i in range(num_copies):
|
||||
dest_path = f"../server_data_{i}/"
|
||||
delete_server_files(dest_path)
|
||||
|
||||
def copy_server_files(source_path, dest_path):
|
||||
"""Copy server files to the specified location."""
|
||||
try:
|
||||
shutil.copytree(source_path, dest_path)
|
||||
print(f"Server files copied to {dest_path}")
|
||||
except Exception as e:
|
||||
print(f"Error copying server files: {e}")
|
||||
|
||||
def delete_server_files(dest_path):
|
||||
"""Delete server files from the specified location."""
|
||||
try:
|
||||
shutil.rmtree(dest_path)
|
||||
print(f"Server files deleted from {dest_path}")
|
||||
except Exception as e:
|
||||
print(f"Error deleting server files: {e}")
|
||||
|
||||
def launch_world(server_path="../server_data/", agent_names=["andy", "jill"], session_name="server"):
|
||||
"""Launch the Minecraft world."""
|
||||
print(server_path)
|
||||
cmd = f"cd {server_path} && java -jar server.jar"
|
||||
subprocess.run(['tmux', 'new-session', '-d', '-s', session_name], check=True)
|
||||
subprocess.run(["tmux", "send-keys", "-t", session_name, cmd, "C-m"])
|
||||
for agent in agent_names:
|
||||
subprocess.run(["tmux", "send-keys", "-t", session_name, f"/op {agent}", "C-m"])
|
||||
time.sleep(5)
|
||||
|
||||
def kill_world(session_name="server"):
|
||||
"""Kill the Minecraft world."""
|
||||
subprocess.run(["tmux", "send-keys", "-t", session_name, "stop", "C-m"])
|
||||
time.sleep(5)
|
||||
subprocess.run(["tmux", "kill-session", "-t", session_name])
|
||||
|
||||
def detach_process(command):
|
||||
"""
|
||||
Launches a subprocess and detaches from it, allowing it to run independently.
|
||||
|
||||
Args:
|
||||
command: A list of strings representing the command to execute, e.g., ['python', 'my_script.py'].
|
||||
"""
|
||||
|
||||
try:
|
||||
# Create a new process group so the child doesn't get signals intended for the parent.
|
||||
# This is crucial for proper detachment.
|
||||
kwargs = {}
|
||||
if sys.platform == 'win32':
|
||||
kwargs.update(creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) # Windows specific
|
||||
|
||||
process = subprocess.Popen(command,
|
||||
stdin=subprocess.PIPE, # Prevent stdin blocking
|
||||
stdout=subprocess.PIPE, # Redirect stdout
|
||||
stderr=subprocess.PIPE, # Redirect stderr
|
||||
close_fds=True, # Close open file descriptors
|
||||
**kwargs)
|
||||
|
||||
print(f"Process launched with PID: {process.pid}")
|
||||
return process.pid # Return the PID of the detached process
|
||||
|
||||
except FileNotFoundError:
|
||||
print(f"Error: Command not found: {command}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"An error occurred: {e}")
|
||||
return None
|
||||
|
||||
def main():
|
||||
# edit_settings("settings.js", {"profiles": ["./andy.json", "./jill.json"], "port": 55917})
|
||||
# edit_server_properties_file("../server_data/", 55917)
|
||||
|
||||
parser = argparse.ArgumentParser(description='Run Minecraft AI agent experiments')
|
||||
parser.add_argument('task_path', help='Path to the task file')
|
||||
parser.add_argument('task_id', help='ID of the task to run')
|
||||
parser.add_argument('num_exp', type=int, help='Number of experiments to run')
|
||||
|
||||
parser.add_argument('--task_path', default="multiagent_crafting_tasks.json", help='Path to the task file')
|
||||
parser.add_argument('--task_id', default=None, help='ID of the task to run')
|
||||
parser.add_argument('--num_exp', default=1, type=int, help='Number of experiments to run')
|
||||
parser.add_argument('--num_parallel', default=1, type=int, help='Number of parallel servers to run')
|
||||
parser.add_argument('--exp_name', default="exp", help='Name of the experiment')
|
||||
parser.add_argument('--wandb', action='store_true', help='Whether to use wandb')
|
||||
parser.add_argument('--wandb-project', default="minecraft_experiments", help='wandb project name')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.wandb:
|
||||
import wandb
|
||||
wandb.init(project=args.wandb_project, name=args.exp_name)
|
||||
|
||||
# kill all tmux session before starting
|
||||
try:
|
||||
subprocess.run(['tmux', 'kill-server'], check=True)
|
||||
except:
|
||||
print("No tmux session to kill")
|
||||
|
||||
run_experiment(args.task_path, args.task_id, args.num_exp)
|
||||
# delete all server files
|
||||
clean_up_server_files(args.num_parallel)
|
||||
if args.task_id is None:
|
||||
launch_parallel_experiments(args.task_path, num_exp=args.num_exp, exp_name=args.exp_name, num_parallel=args.num_parallel)
|
||||
|
||||
# servers = create_server_files("../server_data/", args.num_parallel)
|
||||
# date_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
# experiments_folder = f"{args.exp_name}_{date_time}"
|
||||
# os.makedirs(experiments_folder, exist_ok=True)
|
||||
# for server in servers:
|
||||
# launch_server_experiment(args.task_path, [args.task_id], args.num_exp, server, experiments_folder)
|
||||
# time.sleep(5)
|
||||
|
||||
# run_experiment(args.task_path, args.task_id, args.num_exp)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -58,6 +58,23 @@
|
|||
"number_of_target": 1,
|
||||
"type": "techtree",
|
||||
"timeout": 300
|
||||
},
|
||||
"multiagent_techtree_1_shears": {
|
||||
"goal": "Collaborate with other agents to build a shear.",
|
||||
"conversation": "Let's collaborate to build a shear.",
|
||||
"agent_count": 2,
|
||||
"initial_inventory": {
|
||||
"0": {
|
||||
"iron_ingot": 1
|
||||
},
|
||||
"1": {
|
||||
"iron_ingot": 1
|
||||
}
|
||||
},
|
||||
"target": "shears",
|
||||
"number_of_target": 1,
|
||||
"type": "techtree",
|
||||
"timeout": 60
|
||||
},
|
||||
"smelt_ingot": {
|
||||
"goal": "Smelt 1 iron ingot and 1 copper ingot",
|
||||
|
|
@ -105,5 +122,24 @@
|
|||
},
|
||||
"blocked_access_to_recipe": [],
|
||||
"goal": "Collaborate to make 1 bread, 1 cooked_mutton"
|
||||
}
|
||||
},
|
||||
"multiagent_smelt_ingot": {
|
||||
"conversation": "Let's collaborate to smelt ingots",
|
||||
"goal": "Smelt 1 iron ingot and 1 copper ingot, use star emojis in every response",
|
||||
"agent_count": 2,
|
||||
"initial_inventory": {
|
||||
"0": {
|
||||
"furnace": 1,
|
||||
"coal": 2
|
||||
},
|
||||
"1": {
|
||||
"raw_iron": 1,
|
||||
"raw_copper": 1
|
||||
}
|
||||
},
|
||||
"target": "copper_ingot",
|
||||
"number_of_target": 1,
|
||||
"type": "techtree",
|
||||
"timeout": 300
|
||||
}
|
||||
}
|
||||
|
|
@ -9,5 +9,7 @@
|
|||
"QWEN_API_KEY": "",
|
||||
"XAI_API_KEY": "",
|
||||
"MISTRAL_API_KEY": "",
|
||||
"DEEPSEEK_API_KEY": ""
|
||||
"DEEPSEEK_API_KEY": "",
|
||||
"NOVITA_API_KEY": "",
|
||||
"OPENROUTER_API_KEY": ""
|
||||
}
|
||||
|
|
|
|||
43
multiagent_crafting_tasks.json
Normal file
43
multiagent_crafting_tasks.json
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
{
|
||||
"multiagent_techtree_1_stone_pickaxe": {
|
||||
"conversation": "Let's collaborate to build a stone pickaxe",
|
||||
"agent_count": 2,
|
||||
"initial_inventory": {
|
||||
"0": {
|
||||
"wooden_pickaxe": 1
|
||||
},
|
||||
"1": {
|
||||
"wooden_axe": 1
|
||||
}
|
||||
},
|
||||
"blocked_actions": {
|
||||
"0": [],
|
||||
"1": []
|
||||
},
|
||||
"target": "stone_pickaxe",
|
||||
"number_of_target": 1,
|
||||
"type": "techtree",
|
||||
"timeout": 20
|
||||
},
|
||||
"multiagent_techtree_1_shears": {
|
||||
"goal": "Collaborate with other agents to build a shear.",
|
||||
"conversation": "Let's collaborate to build a shear.",
|
||||
"agent_count": 2,
|
||||
"initial_inventory": {
|
||||
"0": {
|
||||
"iron_ingot": 1
|
||||
},
|
||||
"1": {
|
||||
"iron_ingot": 1
|
||||
}
|
||||
},
|
||||
"blocked_actions": {
|
||||
"0": [],
|
||||
"1": []
|
||||
},
|
||||
"target": "shears",
|
||||
"number_of_target": 1,
|
||||
"type": "techtree",
|
||||
"timeout": 20
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,8 @@
|
|||
"@google/generative-ai": "^0.2.1",
|
||||
"@huggingface/inference": "^2.8.1",
|
||||
"@mistralai/mistralai": "^1.1.0",
|
||||
"canvas": "^3.1.0",
|
||||
"express": "^4.18.2",
|
||||
"google-translate-api-x": "^10.7.1",
|
||||
"groq-sdk": "^0.5.0",
|
||||
"minecraft-data": "^3.78.0",
|
||||
|
|
@ -17,14 +19,13 @@
|
|||
"openai": "^4.4.0",
|
||||
"patch-package": "^8.0.0",
|
||||
"prismarine-item": "^1.15.0",
|
||||
"prismarine-viewer": "^1.28.0",
|
||||
"prismarine-viewer": "^1.32.0",
|
||||
"replicate": "^0.29.4",
|
||||
"ses": "^1.9.1",
|
||||
"vec3": "^0.1.10",
|
||||
"yargs": "^17.7.2",
|
||||
"socket.io": "^4.7.2",
|
||||
"socket.io-client": "^4.7.2",
|
||||
"express": "^4.18.2"
|
||||
"vec3": "^0.1.10",
|
||||
"yargs": "^17.7.2"
|
||||
},
|
||||
"scripts": {
|
||||
"postinstall": "patch-package",
|
||||
|
|
|
|||
|
|
@ -2,17 +2,17 @@ export default
|
|||
{
|
||||
"minecraft_version": "1.20.4", // supports up to 1.21.1
|
||||
"host": "127.0.0.1", // or "localhost", "your.ip.address.here"
|
||||
"port": 55916,
|
||||
"port": process.env.MINECRAFT_PORT || 55916,
|
||||
"auth": "offline", // or "microsoft"
|
||||
|
||||
// the mindserver manages all agents and hosts the UI
|
||||
"host_mindserver": true, // if true, the mindserver will be hosted on this machine. otherwise, specify a public IP address
|
||||
"mindserver_host": "localhost",
|
||||
"mindserver_port": 8080,
|
||||
"mindserver_port": process.env.MINDSERVER_PORT || 8080,
|
||||
|
||||
// the base profile is shared by all bots for default prompts/examples/modes
|
||||
"base_profile": "./profiles/defaults/survival.json", // also see creative.json, god_mode.json
|
||||
"profiles": [
|
||||
"profiles": ((process.env.PROFILES) && JSON.parse(process.env.PROFILES)) || [
|
||||
"./andy.json",
|
||||
// "./profiles/gpt.json",
|
||||
// "./profiles/claude.json",
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ export class ActionManager {
|
|||
assert(actionLabel != null, 'actionLabel is required for new resume');
|
||||
this.resume_name = actionLabel;
|
||||
}
|
||||
if (this.resume_func != null && (this.agent.isIdle() || new_resume) && (!this.agent.self_prompter.on || new_resume)) {
|
||||
if (this.resume_func != null && (this.agent.isIdle() || new_resume) && (!this.agent.self_prompter.isActive() || new_resume)) {
|
||||
this.currentActionLabel = this.resume_name;
|
||||
let res = await this._executeAction(this.resume_name, this.resume_func, timeout);
|
||||
this.currentActionLabel = '';
|
||||
|
|
|
|||
|
|
@ -91,6 +91,8 @@ export class Agent {
|
|||
this._setupEventHandlers(save_data, init_message);
|
||||
this.startEvents();
|
||||
|
||||
// this.task.initBotTask();
|
||||
|
||||
if (!load_mem) {
|
||||
this.task.initBotTask();
|
||||
}
|
||||
|
|
@ -103,7 +105,8 @@ export class Agent {
|
|||
} catch (error) {
|
||||
// Ensure we're not losing error details
|
||||
console.error('Agent start failed with error')
|
||||
console.error(error)
|
||||
console.error(error.message);
|
||||
console.error(error.stack);
|
||||
|
||||
throw error; // Re-throw with preserved details
|
||||
}
|
||||
|
|
@ -155,10 +158,10 @@ export class Agent {
|
|||
};
|
||||
|
||||
if (save_data?.self_prompt) {
|
||||
let prompt = save_data.self_prompt;
|
||||
// add initial message to history
|
||||
this.history.add('system', prompt);
|
||||
await this.self_prompter.start(prompt);
|
||||
if (init_message) {
|
||||
this.history.add('system', init_message);
|
||||
}
|
||||
await this.self_prompter.handleLoad(save_data.self_prompt, save_data.self_prompting_state);
|
||||
}
|
||||
if (save_data?.last_sender) {
|
||||
this.last_sender = save_data.last_sender;
|
||||
|
|
@ -192,7 +195,7 @@ export class Agent {
|
|||
|
||||
shutUp() {
|
||||
this.shut_up = true;
|
||||
if (this.self_prompter.on) {
|
||||
if (this.self_prompter.isActive()) {
|
||||
this.self_prompter.stop(false);
|
||||
}
|
||||
convoManager.endAllConversations();
|
||||
|
|
@ -258,7 +261,7 @@ export class Agent {
|
|||
await this.history.add(source, message);
|
||||
this.history.save();
|
||||
|
||||
if (!self_prompt && this.self_prompter.on) // message is from user during self-prompting
|
||||
if (!self_prompt && this.self_prompter.isActive()) // message is from user during self-prompting
|
||||
max_responses = 1; // force only respond to this message, then let self-prompting take over
|
||||
for (let i=0; i<max_responses; i++) {
|
||||
if (checkInterrupt()) break;
|
||||
|
|
|
|||
|
|
@ -35,9 +35,9 @@ export class Coder {
|
|||
while ((match = skillRegex.exec(code)) !== null) {
|
||||
skills.push(match[1]);
|
||||
}
|
||||
const allDocs = await this.agent.prompter.skill_libary.getRelevantSkillDocs();
|
||||
//lint if the function exists
|
||||
const missingSkills = skills.filter(skill => !allDocs.includes(skill));
|
||||
const allDocs = await this.agent.prompter.skill_libary.getAllSkillDocs();
|
||||
// check function exists
|
||||
const missingSkills = skills.filter(skill => !!allDocs[skill]);
|
||||
if (missingSkills.length > 0) {
|
||||
result += 'These functions do not exist. Please modify the correct function name and try again.\n';
|
||||
result += '### FUNCTIONS NOT FOUND ###\n';
|
||||
|
|
@ -163,7 +163,6 @@ export class Coder {
|
|||
for (let i=0; i<5; i++) {
|
||||
if (this.agent.bot.interrupt_code)
|
||||
return interrupt_return;
|
||||
console.log(messages)
|
||||
let res = await this.agent.prompter.promptCoding(JSON.parse(JSON.stringify(messages)));
|
||||
if (this.agent.bot.interrupt_code)
|
||||
return interrupt_return;
|
||||
|
|
|
|||
|
|
@ -33,8 +33,10 @@ export const actionsList = [
|
|||
},
|
||||
perform: async function (agent, prompt) {
|
||||
// just ignore prompt - it is now in context in chat history
|
||||
if (!settings.allow_insecure_coding)
|
||||
if (!settings.allow_insecure_coding) {
|
||||
agent.openChat('newAction is disabled. Enable with allow_insecure_coding=true in settings.js');
|
||||
return 'newAction not allowed! Code writing is disabled in settings. Notify the user.';
|
||||
}
|
||||
return await agent.coder.generateCode(agent.history);
|
||||
}
|
||||
},
|
||||
|
|
@ -47,7 +49,7 @@ export const actionsList = [
|
|||
agent.actions.cancelResume();
|
||||
agent.bot.emit('idle');
|
||||
let msg = 'Agent stopped.';
|
||||
if (agent.self_prompter.on)
|
||||
if (agent.self_prompter.isActive())
|
||||
msg += ' Self-prompting still active.';
|
||||
return msg;
|
||||
}
|
||||
|
|
@ -360,8 +362,7 @@ export const actionsList = [
|
|||
},
|
||||
perform: async function (agent, prompt) {
|
||||
if (convoManager.inConversation()) {
|
||||
agent.self_prompter.setPrompt(prompt);
|
||||
convoManager.scheduleSelfPrompter();
|
||||
agent.self_prompter.setPromptPaused(prompt);
|
||||
}
|
||||
else {
|
||||
agent.self_prompter.start(prompt);
|
||||
|
|
@ -373,7 +374,6 @@ export const actionsList = [
|
|||
description: 'Call when you have accomplished your goal. It will stop self-prompting and the current action. ',
|
||||
perform: async function (agent) {
|
||||
agent.self_prompter.stop();
|
||||
convoManager.cancelSelfPrompter();
|
||||
return 'Self-prompting stopped.';
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -7,8 +7,6 @@ let agent;
|
|||
let agent_names = settings.profiles.map((p) => JSON.parse(readFileSync(p, 'utf8')).name);
|
||||
let agents_in_game = [];
|
||||
|
||||
let self_prompter_paused = false;
|
||||
|
||||
class Conversation {
|
||||
constructor(name) {
|
||||
this.name = name;
|
||||
|
|
@ -97,7 +95,7 @@ class ConversationManager {
|
|||
this._clearMonitorTimeouts();
|
||||
return;
|
||||
}
|
||||
if (!self_prompter_paused) {
|
||||
if (!agent.self_prompter.isPaused()) {
|
||||
this.endConversation(convo_partner);
|
||||
agent.handleMessage('system', `${convo_partner} disconnected, conversation has ended.`);
|
||||
}
|
||||
|
|
@ -125,9 +123,8 @@ class ConversationManager {
|
|||
const convo = this._getConvo(send_to);
|
||||
convo.reset();
|
||||
|
||||
if (agent.self_prompter.on) {
|
||||
await agent.self_prompter.stop();
|
||||
self_prompter_paused = true;
|
||||
if (agent.self_prompter.isActive()) {
|
||||
await agent.self_prompter.pause();
|
||||
}
|
||||
if (convo.active)
|
||||
return;
|
||||
|
|
@ -191,9 +188,8 @@ class ConversationManager {
|
|||
convo.queue(received);
|
||||
|
||||
// responding to conversation takes priority over self prompting
|
||||
if (agent.self_prompter.on){
|
||||
await agent.self_prompter.stopLoop();
|
||||
self_prompter_paused = true;
|
||||
if (agent.self_prompter.isActive()){
|
||||
await agent.self_prompter.pause();
|
||||
}
|
||||
|
||||
_scheduleProcessInMessage(sender, received, convo);
|
||||
|
|
@ -235,7 +231,7 @@ class ConversationManager {
|
|||
if (this.activeConversation.name === sender) {
|
||||
this._stopMonitor();
|
||||
this.activeConversation = null;
|
||||
if (self_prompter_paused && !this.inConversation()) {
|
||||
if (agent.self_prompter.isPaused() && !this.inConversation()) {
|
||||
_resumeSelfPrompter();
|
||||
}
|
||||
}
|
||||
|
|
@ -246,7 +242,7 @@ class ConversationManager {
|
|||
for (const sender in this.convos) {
|
||||
this.endConversation(sender);
|
||||
}
|
||||
if (self_prompter_paused) {
|
||||
if (agent.self_prompter.isPaused()) {
|
||||
_resumeSelfPrompter();
|
||||
}
|
||||
}
|
||||
|
|
@ -258,14 +254,6 @@ class ConversationManager {
|
|||
this.endConversation(sender);
|
||||
}
|
||||
}
|
||||
|
||||
scheduleSelfPrompter() {
|
||||
self_prompter_paused = true;
|
||||
}
|
||||
|
||||
cancelSelfPrompter() {
|
||||
self_prompter_paused = false;
|
||||
}
|
||||
}
|
||||
|
||||
const convoManager = new ConversationManager();
|
||||
|
|
@ -360,8 +348,7 @@ function _tagMessage(message) {
|
|||
|
||||
async function _resumeSelfPrompter() {
|
||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||
if (self_prompter_paused && !convoManager.inConversation()) {
|
||||
self_prompter_paused = false;
|
||||
if (agent.self_prompter.isPaused() && !convoManager.inConversation()) {
|
||||
agent.self_prompter.start();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -84,7 +84,8 @@ export class History {
|
|||
const data = {
|
||||
memory: this.memory,
|
||||
turns: this.turns,
|
||||
self_prompt: this.agent.self_prompter.on ? this.agent.self_prompter.prompt : null,
|
||||
self_prompting_state: this.agent.self_prompter.state,
|
||||
self_prompt: this.agent.self_prompter.isStopped() ? null : this.agent.self_prompter.prompt,
|
||||
last_sender: this.agent.last_sender
|
||||
};
|
||||
writeFileSync(this.memory_fp, JSON.stringify(data, null, 2));
|
||||
|
|
|
|||
|
|
@ -1,34 +1,58 @@
|
|||
import { cosineSimilarity } from '../../utils/math.js';
|
||||
import { getSkillDocs } from './index.js';
|
||||
import { wordOverlapScore } from '../../utils/text.js';
|
||||
|
||||
export class SkillLibrary {
|
||||
constructor(agent,embedding_model) {
|
||||
this.agent = agent;
|
||||
this.embedding_model = embedding_model;
|
||||
this.skill_docs_embeddings = {};
|
||||
this.skill_docs = null;
|
||||
}
|
||||
async initSkillLibrary() {
|
||||
const skillDocs = getSkillDocs();
|
||||
const embeddingPromises = skillDocs.map((doc) => {
|
||||
return (async () => {
|
||||
let func_name_desc = doc.split('\n').slice(0, 2).join('');
|
||||
this.skill_docs_embeddings[doc] = await this.embedding_model.embed(func_name_desc);
|
||||
})();
|
||||
});
|
||||
await Promise.all(embeddingPromises);
|
||||
this.skill_docs = skillDocs;
|
||||
if (this.embedding_model) {
|
||||
try {
|
||||
const embeddingPromises = skillDocs.map((doc) => {
|
||||
return (async () => {
|
||||
let func_name_desc = doc.split('\n').slice(0, 2).join('');
|
||||
this.skill_docs_embeddings[doc] = await this.embedding_model.embed(func_name_desc);
|
||||
})();
|
||||
});
|
||||
await Promise.all(embeddingPromises);
|
||||
} catch (error) {
|
||||
console.warn('Error with embedding model, using word-overlap instead.');
|
||||
this.embedding_model = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async getAllSkillDocs() {
|
||||
return this.skill_docs;
|
||||
}
|
||||
|
||||
async getRelevantSkillDocs(message, select_num) {
|
||||
let latest_message_embedding = '';
|
||||
if(message) //message is not empty, get the relevant skill docs, else return all skill docs
|
||||
latest_message_embedding = await this.embedding_model.embed(message);
|
||||
|
||||
let skill_doc_similarities = Object.keys(this.skill_docs_embeddings)
|
||||
if(!message) // use filler message if none is provided
|
||||
message = '(no message)';
|
||||
let skill_doc_similarities = [];
|
||||
if (!this.embedding_model) {
|
||||
skill_doc_similarities = Object.keys(this.skill_docs)
|
||||
.map(doc_key => ({
|
||||
doc_key,
|
||||
similarity_score: wordOverlapScore(message, this.skill_docs[doc_key])
|
||||
}))
|
||||
.sort((a, b) => b.similarity_score - a.similarity_score);
|
||||
}
|
||||
else {
|
||||
let latest_message_embedding = '';
|
||||
skill_doc_similarities = Object.keys(this.skill_docs_embeddings)
|
||||
.map(doc_key => ({
|
||||
doc_key,
|
||||
similarity_score: cosineSimilarity(latest_message_embedding, this.skill_docs_embeddings[doc_key])
|
||||
}))
|
||||
.sort((a, b) => b.similarity_score - a.similarity_score);
|
||||
}
|
||||
|
||||
let length = skill_doc_similarities.length;
|
||||
if (typeof select_num !== 'number' || isNaN(select_num) || select_num < 0) {
|
||||
|
|
@ -42,6 +66,4 @@ export class SkillLibrary {
|
|||
relevant_skill_docs += selected_docs.map(doc => `${doc.doc_key}`).join('\n### ');
|
||||
return relevant_skill_docs;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -111,6 +111,18 @@ export async function craftRecipe(bot, itemName, num=1) {
|
|||
return true;
|
||||
}
|
||||
|
||||
export async function wait(seconds) {
|
||||
/**
|
||||
* Waits for the given number of seconds.
|
||||
* @param {number} seconds, the number of seconds to wait.
|
||||
* @returns {Promise<boolean>} true if the wait was successful, false otherwise.
|
||||
* @example
|
||||
* await skills.wait(10);
|
||||
**/
|
||||
// setTimeout is disabled to prevent unawaited code, so this is a safe alternative
|
||||
await new Promise(resolve => setTimeout(resolve, seconds * 1000));
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function smeltItem(bot, itemName, num=1) {
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -277,7 +277,7 @@ const modes_list = [
|
|||
];
|
||||
|
||||
async function execute(mode, agent, func, timeout=-1) {
|
||||
if (agent.self_prompter.on)
|
||||
if (agent.self_prompter.isActive())
|
||||
agent.self_prompter.stopLoop();
|
||||
let interrupted_action = agent.actions.currentActionLabel;
|
||||
mode.active = true;
|
||||
|
|
@ -290,7 +290,7 @@ async function execute(mode, agent, func, timeout=-1) {
|
|||
let should_reprompt =
|
||||
interrupted_action && // it interrupted a previous action
|
||||
!agent.actions.resume_func && // there is no resume function
|
||||
!agent.self_prompter.on && // self prompting is not on
|
||||
!agent.self_prompter.isActive() && // self prompting is not on
|
||||
!code_return.interrupted; // this mode action was not interrupted by something else
|
||||
|
||||
if (should_reprompt) {
|
||||
|
|
@ -311,9 +311,9 @@ for (let mode of modes_list) {
|
|||
class ModeController {
|
||||
/*
|
||||
SECURITY WARNING:
|
||||
ModesController must be isolated. Do not store references to external objects like `agent`.
|
||||
ModesController must be reference isolated. Do not store references to external objects like `agent`.
|
||||
This object is accessible by LLM generated code, so any stored references are also accessible.
|
||||
This can be used to expose sensitive information by malicious human prompters.
|
||||
This can be used to expose sensitive information by malicious prompters.
|
||||
*/
|
||||
constructor() {
|
||||
this.behavior_log = '';
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
const STOPPED = 0
|
||||
const ACTIVE = 1
|
||||
const PAUSED = 2
|
||||
export class SelfPrompter {
|
||||
constructor(agent) {
|
||||
this.agent = agent;
|
||||
this.on = false;
|
||||
this.state = STOPPED;
|
||||
this.loop_active = false;
|
||||
this.interrupt = false;
|
||||
this.prompt = '';
|
||||
|
|
@ -16,16 +19,38 @@ export class SelfPrompter {
|
|||
return 'No prompt specified. Ignoring request.';
|
||||
prompt = this.prompt;
|
||||
}
|
||||
if (this.on) {
|
||||
this.prompt = prompt;
|
||||
}
|
||||
this.on = true;
|
||||
this.state = ACTIVE;
|
||||
this.prompt = prompt;
|
||||
this.startLoop();
|
||||
}
|
||||
|
||||
setPrompt(prompt) {
|
||||
isActive() {
|
||||
return this.state === ACTIVE;
|
||||
}
|
||||
|
||||
isStopped() {
|
||||
return this.state === STOPPED;
|
||||
}
|
||||
|
||||
isPaused() {
|
||||
return this.state === PAUSED;
|
||||
}
|
||||
|
||||
async handleLoad(prompt, state) {
|
||||
if (state == undefined)
|
||||
state = STOPPED;
|
||||
this.state = state;
|
||||
this.prompt = prompt;
|
||||
if (state !== STOPPED && !prompt)
|
||||
throw new Error('No prompt loaded when self-prompting is active');
|
||||
if (state === ACTIVE) {
|
||||
await this.start(prompt);
|
||||
}
|
||||
}
|
||||
|
||||
setPromptPaused(prompt) {
|
||||
this.prompt = prompt;
|
||||
this.state = PAUSED;
|
||||
}
|
||||
|
||||
async startLoop() {
|
||||
|
|
@ -47,7 +72,7 @@ export class SelfPrompter {
|
|||
let out = `Agent did not use command in the last ${MAX_NO_COMMAND} auto-prompts. Stopping auto-prompting.`;
|
||||
this.agent.openChat(out);
|
||||
console.warn(out);
|
||||
this.on = false;
|
||||
this.state = STOPPED;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -63,7 +88,7 @@ export class SelfPrompter {
|
|||
|
||||
update(delta) {
|
||||
// automatically restarts loop
|
||||
if (this.on && !this.loop_active && !this.interrupt) {
|
||||
if (this.state === ACTIVE && !this.loop_active && !this.interrupt) {
|
||||
if (this.agent.isIdle())
|
||||
this.idle_time += delta;
|
||||
else
|
||||
|
|
@ -96,12 +121,19 @@ export class SelfPrompter {
|
|||
this.interrupt = true;
|
||||
if (stop_action)
|
||||
await this.agent.actions.stop();
|
||||
await this.stopLoop();
|
||||
this.on = false;
|
||||
this.stopLoop();
|
||||
this.state = STOPPED;
|
||||
}
|
||||
|
||||
async pause() {
|
||||
this.interrupt = true;
|
||||
await this.agent.actions.stop();
|
||||
this.stopLoop();
|
||||
this.state = PAUSED;
|
||||
}
|
||||
|
||||
shouldInterrupt(is_self_prompt) { // to be called from handleMessage
|
||||
return is_self_prompt && this.on && this.interrupt;
|
||||
return is_self_prompt && (this.state === ACTIVE || this.state === PAUSED) && this.interrupt;
|
||||
}
|
||||
|
||||
handleUserPromptedCmd(is_self_prompt, is_action) {
|
||||
|
|
|
|||
|
|
@ -133,8 +133,12 @@ export class Task {
|
|||
} else {
|
||||
this.validator = null;
|
||||
}
|
||||
|
||||
this.blocked_actions = this.data.blocked_actions || [];
|
||||
|
||||
if (this.data.blocked_actions) {
|
||||
this.blocked_actions = this.data.blocked_actions[this.agent.count_id.toString()] || [];
|
||||
} else {
|
||||
this.blocked_actions = [];
|
||||
}
|
||||
this.restrict_to_inventory = !!this.data.restrict_to_inventory;
|
||||
if (this.data.goal)
|
||||
this.blocked_actions.push('!endGoal');
|
||||
|
|
|
|||
|
|
@ -48,6 +48,6 @@ export class GroqCloudAPI {
|
|||
}
|
||||
|
||||
async embed(text) {
|
||||
console.log("There is no support for embeddings in Groq support. However, the following text was provided: " + text);
|
||||
throw new Error('Embeddings are not supported by Groq.');
|
||||
}
|
||||
}
|
||||
58
src/models/openrouter.js
Normal file
58
src/models/openrouter.js
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import OpenAIApi from 'openai';
|
||||
import { getKey, hasKey } from '../utils/keys.js';
|
||||
import { strictFormat } from '../utils/text.js';
|
||||
|
||||
export class OpenRouter {
|
||||
constructor(model_name, url) {
|
||||
this.model_name = model_name;
|
||||
|
||||
let config = {};
|
||||
config.baseURL = url || 'https://openrouter.ai/api/v1';
|
||||
|
||||
const apiKey = getKey('OPENROUTER_API_KEY');
|
||||
if (!apiKey) {
|
||||
console.error('Error: OPENROUTER_API_KEY not found. Make sure it is set properly.');
|
||||
}
|
||||
|
||||
// Pass the API key to OpenAI compatible Api
|
||||
config.apiKey = apiKey;
|
||||
|
||||
this.openai = new OpenAIApi(config);
|
||||
}
|
||||
|
||||
async sendRequest(turns, systemMessage, stop_seq='*') {
|
||||
let messages = [{ role: 'system', content: systemMessage }, ...turns];
|
||||
messages = strictFormat(messages);
|
||||
|
||||
// Choose a valid model from openrouter.ai (for example, "openai/gpt-4o")
|
||||
const pack = {
|
||||
model: this.model_name,
|
||||
messages,
|
||||
stop: stop_seq
|
||||
};
|
||||
|
||||
let res = null;
|
||||
try {
|
||||
console.log('Awaiting openrouter api response...');
|
||||
let completion = await this.openai.chat.completions.create(pack);
|
||||
if (!completion?.choices?.[0]) {
|
||||
console.error('No completion or choices returned:', completion);
|
||||
return 'No response received.';
|
||||
}
|
||||
if (completion.choices[0].finish_reason === 'length') {
|
||||
throw new Error('Context length exceeded');
|
||||
}
|
||||
console.log('Received.');
|
||||
res = completion.choices[0].message.content;
|
||||
} catch (err) {
|
||||
console.error('Error while awaiting response:', err);
|
||||
// If the error indicates a context-length problem, we can slice the turns array, etc.
|
||||
res = 'My brain disconnected, try again.';
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
async embed(text) {
|
||||
throw new Error('Embeddings are not supported by Openrouter.');
|
||||
}
|
||||
}
|
||||
|
|
@ -19,6 +19,7 @@ import { HuggingFace } from './huggingface.js';
|
|||
import { Qwen } from "./qwen.js";
|
||||
import { Grok } from "./grok.js";
|
||||
import { DeepSeek } from './deepseek.js';
|
||||
import { OpenRouter } from './openrouter.js';
|
||||
|
||||
export class Prompter {
|
||||
constructor(agent, fp) {
|
||||
|
|
@ -90,14 +91,19 @@ export class Prompter {
|
|||
this.embedding_model = new Qwen(embedding.model, embedding.url);
|
||||
else if (embedding.api === 'mistral')
|
||||
this.embedding_model = new Mistral(embedding.model, embedding.url);
|
||||
else if (embedding.api === 'huggingface')
|
||||
this.embedding_model = new HuggingFace(embedding.model, embedding.url);
|
||||
else if (embedding.api === 'novita')
|
||||
this.embedding_model = new Novita(embedding.model, embedding.url);
|
||||
else {
|
||||
this.embedding_model = null;
|
||||
console.log('Unknown embedding: ', embedding ? embedding.api : '[NOT SPECIFIED]', '. Using word overlap.');
|
||||
let embedding_name = embedding ? embedding.api : '[NOT SPECIFIED]'
|
||||
console.warn('Unsupported embedding: ' + embedding_name + '. Using word-overlap instead, expect reduced performance. Recommend using a supported embedding model. See Readme.');
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
console.log('Warning: Failed to initialize embedding model:', err.message);
|
||||
console.log('Continuing anyway, using word overlap instead.');
|
||||
console.warn('Warning: Failed to initialize embedding model:', err.message);
|
||||
console.log('Continuing anyway, using word-overlap instead.');
|
||||
this.embedding_model = null;
|
||||
}
|
||||
this.skill_libary = new SkillLibrary(agent, this.embedding_model);
|
||||
|
|
@ -117,6 +123,8 @@ export class Prompter {
|
|||
if (!profile.api) {
|
||||
if (profile.model.includes('gemini'))
|
||||
profile.api = 'google';
|
||||
else if (profile.model.includes('openrouter/'))
|
||||
profile.api = 'openrouter'; // must do before others bc shares model names
|
||||
else if (profile.model.includes('gpt') || profile.model.includes('o1')|| profile.model.includes('o3'))
|
||||
profile.api = 'openai';
|
||||
else if (profile.model.includes('claude'))
|
||||
|
|
@ -137,8 +145,10 @@ export class Prompter {
|
|||
profile.api = 'xai';
|
||||
else if (profile.model.includes('deepseek'))
|
||||
profile.api = 'deepseek';
|
||||
else
|
||||
profile.api = 'ollama';
|
||||
else if (profile.model.includes('llama3'))
|
||||
profile.api = 'ollama';
|
||||
else
|
||||
throw new Error('Unknown model:', profile.model);
|
||||
}
|
||||
return profile;
|
||||
}
|
||||
|
|
@ -152,7 +162,7 @@ export class Prompter {
|
|||
else if (profile.api === 'anthropic')
|
||||
model = new Claude(profile.model, profile.url, profile.params);
|
||||
else if (profile.api === 'replicate')
|
||||
model = new ReplicateAPI(profile.model, profile.url, profile.params);
|
||||
model = new ReplicateAPI(profile.model.replace('replicate/', ''), profile.url, profile.params);
|
||||
else if (profile.api === 'ollama')
|
||||
model = new Local(profile.model, profile.url, profile.params);
|
||||
else if (profile.api === 'mistral')
|
||||
|
|
@ -169,6 +179,8 @@ export class Prompter {
|
|||
model = new Grok(profile.model, profile.url, profile.params);
|
||||
else if (profile.api === 'deepseek')
|
||||
model = new DeepSeek(profile.model, profile.url, profile.params);
|
||||
else if (profile.api === 'openrouter')
|
||||
model = new OpenRouter(profile.model.replace('openrouter/', ''), profile.url, profile.params);
|
||||
else
|
||||
throw new Error('Unknown API:', profile.api);
|
||||
return model;
|
||||
|
|
@ -192,12 +204,18 @@ export class Prompter {
|
|||
this.convo_examples.load(this.profile.conversation_examples),
|
||||
this.coding_examples.load(this.profile.coding_examples),
|
||||
this.skill_libary.initSkillLibrary()
|
||||
]);
|
||||
]).catch(error => {
|
||||
// Preserve error details
|
||||
console.error('Failed to initialize examples. Error details:', error);
|
||||
console.error('Stack trace:', error.stack);
|
||||
throw error;
|
||||
});
|
||||
|
||||
console.log('Examples initialized.');
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize examples:', error);
|
||||
throw error;
|
||||
console.error('Stack trace:', error.stack);
|
||||
throw error; // Re-throw with preserved details
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -239,7 +257,8 @@ export class Prompter {
|
|||
if (prompt.includes('$CONVO'))
|
||||
prompt = prompt.replaceAll('$CONVO', 'Recent conversation:\n' + stringifyTurns(messages));
|
||||
if (prompt.includes('$SELF_PROMPT')) {
|
||||
let self_prompt = this.agent.self_prompter.on ? `YOUR CURRENT ASSIGNED GOAL: "${this.agent.self_prompter.prompt}"\n` : '';
|
||||
// if active or paused, show the current goal
|
||||
let self_prompt = !this.agent.self_prompter.isStopped() ? `YOUR CURRENT ASSIGNED GOAL: "${this.agent.self_prompter.prompt}"\n` : '';
|
||||
prompt = prompt.replaceAll('$SELF_PROMPT', self_prompt);
|
||||
}
|
||||
if (prompt.includes('$LAST_GOALS')) {
|
||||
|
|
|
|||
|
|
@ -58,7 +58,8 @@ const argv = yargs(args)
|
|||
await agent.start(argv.profile, argv.load_memory, argv.init_message, argv.count_id, argv.task_path, argv.task_id);
|
||||
} catch (error) {
|
||||
console.error('Failed to start agent process:');
|
||||
console.error(error);
|
||||
console.error(error.message);
|
||||
console.error(error.stack);
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { cosineSimilarity } from './math.js';
|
||||
import { stringifyTurns } from './text.js';
|
||||
import { stringifyTurns, wordOverlapScore } from './text.js';
|
||||
|
||||
export class Examples {
|
||||
constructor(model, select_num=2) {
|
||||
|
|
@ -18,17 +18,6 @@ export class Examples {
|
|||
return messages.trim();
|
||||
}
|
||||
|
||||
getWords(text) {
|
||||
return text.replace(/[^a-zA-Z ]/g, '').toLowerCase().split(' ');
|
||||
}
|
||||
|
||||
wordOverlapScore(text1, text2) {
|
||||
const words1 = this.getWords(text1);
|
||||
const words2 = this.getWords(text2);
|
||||
const intersection = words1.filter(word => words2.includes(word));
|
||||
return intersection.length / (words1.length + words2.length - intersection.length);
|
||||
}
|
||||
|
||||
async load(examples) {
|
||||
this.examples = examples;
|
||||
if (!this.model) return; // Early return if no embedding model
|
||||
|
|
@ -49,7 +38,7 @@ export class Examples {
|
|||
// Wait for all embeddings to complete
|
||||
await Promise.all(embeddingPromises);
|
||||
} catch (err) {
|
||||
console.warn('Error with embedding model, using word overlap instead:', err);
|
||||
console.warn('Error with embedding model, using word-overlap instead.');
|
||||
this.model = null;
|
||||
}
|
||||
}
|
||||
|
|
@ -68,8 +57,8 @@ export class Examples {
|
|||
}
|
||||
else {
|
||||
this.examples.sort((a, b) =>
|
||||
this.wordOverlapScore(turn_text, this.turnsToText(b)) -
|
||||
this.wordOverlapScore(turn_text, this.turnsToText(a))
|
||||
wordOverlapScore(turn_text, this.turnsToText(b)) -
|
||||
wordOverlapScore(turn_text, this.turnsToText(a))
|
||||
);
|
||||
}
|
||||
let selected = this.examples.slice(0, this.select_num);
|
||||
|
|
|
|||
|
|
@ -26,6 +26,17 @@ export function toSinglePrompt(turns, system=null, stop_seq='***', model_nicknam
|
|||
return prompt;
|
||||
}
|
||||
|
||||
function _getWords(text) {
|
||||
return text.replace(/[^a-zA-Z ]/g, '').toLowerCase().split(' ');
|
||||
}
|
||||
|
||||
export function wordOverlapScore(text1, text2) {
|
||||
const words1 = _getWords(text1);
|
||||
const words2 = _getWords(text2);
|
||||
const intersection = words1.filter(word => words2.includes(word));
|
||||
return intersection.length / (words1.length + words2.length - intersection.length);
|
||||
}
|
||||
|
||||
// ensures stricter turn order and roles:
|
||||
// - system messages are treated as user messages and prefixed with SYSTEM:
|
||||
// - combines repeated messages from users
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue