mirror of
https://github.com/kolbytn/mindcraft.git
synced 2025-07-01 14:15:18 +02:00
be38f56f12
This update finalizes the implementation of three distinct vision modes:
- "off": This disables all my vision capabilities.
- "prompted": (Formerly "on") This allows me to use vision via explicit commands from you (e.g., !lookAtPlayer), and I will then summarize the image.
- "always": (Formerly "always_active") I will automatically take a screenshot every time you send a prompt and send it with your prompt to a multimodal LLM. If you use a look command in this mode, I will only update my view and take a screenshot for the *next* interaction if relevant, without immediate summarization.
Here are the key changes and improvements:
1. **Bug Fix (Image Path ENOENT)**:
* I've corrected `Camera.capture()` so it returns filenames with the `.jpg` extension.
* I've updated `VisionInterpreter.analyzeImage()` to handle full filenames.
* This resolves the `ENOENT` error that was previously happening in `Prompter.js`.
2. **Vision Mode Renaming**:
* I've renamed the modes in `settings.js` and throughout the codebase: "on" is now "prompted", and "always_active" is now "always".
3. **Core Framework (from previous work, now integrated)**:
* I've added `vision_mode` to `settings.js`.
* `Agent.js` now manages `latestScreenshotPath` and initializes `VisionInterpreter` with `vision_mode`.
* `VisionInterpreter.js` handles different behaviors for each mode.
* My vision commands (`!lookAt...`) respect the `off` mode.
* `History.js` stores `imagePath` with turns, and `Agent.js` manages this path's lifecycle.
* `Prompter.js` reads image files when I'm in "always" mode and passes `imageData` to model wrappers.
4. **Extended Multimodal API Support**:
* `gemini.js`, `gpt.js`, `claude.js`, `local.js` (Ollama), `qwen.js`, and `deepseek.js` have been updated to accept `imageData` in their `sendRequest` method and format it for their respective multimodal APIs. They now include `supportsRawImageInput = true`.
* Other model wrappers (`mistral.js`, `glhf.js`, `grok.js`, etc.) now safely handle the `imageData` parameter in `sendRequest` (by ignoring it and logging a warning) and have `supportsRawImageInput = false` for that method, ensuring consistent behavior.
5. **Testing**: I have a comprehensive plan to verify all modes and functionalities.
This set of changes provides a robust and flexible vision system for me, catering to different operational needs and supporting various multimodal LLMs.
78 lines
3.7 KiB
JavaScript
78 lines
3.7 KiB
JavaScript
const settings = {
|
|
"minecraft_version": "1.21.1", // supports up to 1.21.1
|
|
"host": "127.0.0.1", // or "localhost", "your.ip.address.here"
|
|
"port": 55916,
|
|
"auth": "offline", // or "microsoft"
|
|
|
|
// the mindserver manages all agents and hosts the UI
|
|
"host_mindserver": true, // if true, the mindserver will be hosted on this machine. otherwise, specify a public IP address
|
|
"mindserver_host": "localhost",
|
|
"mindserver_port": 8080,
|
|
|
|
// the base profile is shared by all bots for default prompts/examples/modes
|
|
"base_profile": "./profiles/defaults/survival.json", // also see creative.json, god_mode.json
|
|
"profiles": [
|
|
"./andy.json",
|
|
// "./profiles/gpt.json",
|
|
// "./profiles/claude.json",
|
|
// "./profiles/gemini.json",
|
|
// "./profiles/llama.json",
|
|
// "./profiles/qwen.json",
|
|
// "./profiles/grok.json",
|
|
// "./profiles/mistral.json",
|
|
// "./profiles/deepseek.json",
|
|
// "./profiles/andy-4.json",
|
|
|
|
// using more than 1 profile requires you to /msg each bot indivually
|
|
// individual profiles override values from the base profile
|
|
],
|
|
"load_memory": false, // load memory from previous session
|
|
"init_message": "Respond with hello world and your name", // sends to all on spawn
|
|
"only_chat_with": [], // users that the bots listen to and send general messages to. if empty it will chat publicly
|
|
"speak": false, // allows all bots to speak through system text-to-speech. works on windows, mac, on linux you need to `apt install espeak`
|
|
"language": "en", // translate to/from this language. Supports these language names: https://cloud.google.com/translate/docs/languages
|
|
"show_bot_views": false, // show bot's view in browser at localhost:3000, 3001...
|
|
|
|
"allow_insecure_coding": false, // allows newAction command and model can write/run code on your computer. enable at own risk
|
|
"allow_vision": false, // allows vision model to interpret screenshots as inputs
|
|
"vision_mode": "prompted", // "off", "prompted", or "always"
|
|
"blocked_actions" : ["!checkBlueprint", "!checkBlueprintLevel", "!getBlueprint", "!getBlueprintLevel"] , // commands to disable and remove from docs. Ex: ["!setMode"]
|
|
"code_timeout_mins": -1, // minutes code is allowed to run. -1 for no timeout
|
|
"relevant_docs_count": 5, // number of relevant code function docs to select for prompting. -1 for all
|
|
|
|
"max_messages": 15, // max number of messages to keep in context
|
|
"num_examples": 2, // number of examples to give to the model
|
|
"max_commands": -1, // max number of commands that can be used in consecutive responses. -1 for no limit
|
|
"verbose_commands": true, // show full command syntax
|
|
"narrate_behavior": true, // chat simple automatic actions ('Picking up item!')
|
|
"chat_bot_messages": true, // publicly chat messages to other bots
|
|
"log_all_prompts": false, // log ALL prompts to file
|
|
}
|
|
|
|
// these environment variables override certain settings
|
|
if (process.env.MINECRAFT_PORT) {
|
|
settings.port = process.env.MINECRAFT_PORT;
|
|
}
|
|
if (process.env.MINDSERVER_PORT) {
|
|
settings.mindserver_port = process.env.MINDSERVER_PORT;
|
|
}
|
|
if (process.env.PROFILES && JSON.parse(process.env.PROFILES).length > 0) {
|
|
settings.profiles = JSON.parse(process.env.PROFILES);
|
|
}
|
|
if (process.env.INSECURE_CODING) {
|
|
settings.allow_insecure_coding = true;
|
|
}
|
|
if (process.env.BLOCKED_ACTIONS) {
|
|
settings.blocked_actions = JSON.parse(process.env.BLOCKED_ACTIONS);
|
|
}
|
|
if (process.env.MAX_MESSAGES) {
|
|
settings.max_messages = process.env.MAX_MESSAGES;
|
|
}
|
|
if (process.env.NUM_EXAMPLES) {
|
|
settings.num_examples = process.env.NUM_EXAMPLES;
|
|
}
|
|
if (process.env.LOG_ALL) {
|
|
settings.log_all_prompts = process.env.LOG_ALL;
|
|
}
|
|
|
|
export default settings;
|