mirror of
https://github.com/kolbytn/mindcraft.git
synced 2025-07-01 06:05:19 +02:00
feat: update skill look to lookAtPlayer & export lookAtPosition
This commit is contained in:
gmuffiness
parent
1be24f4867
commit
f5923db43a
5 changed files with 116 additions and 124 deletions
|
|
@ -25,8 +25,8 @@
|
|||
"socket.io": "^4.7.2",
|
||||
"socket.io-client": "^4.7.2",
|
||||
"express": "^4.18.2",
|
||||
"three": "0.128.0",
|
||||
"node-canvas-webgl": "PrismarineJS/node-canvas-webgl"
|
||||
"node-canvas-webgl": "^0.2.6",
|
||||
"three": "^0.128.0"
|
||||
},
|
||||
"scripts": {
|
||||
"postinstall": "patch-package",
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@
|
|||
|
||||
"bot_responder": "You are a minecraft bot named $NAME that is currently in conversation with another AI bot. Both of you can take actions with the !command syntax, and actions take time to complete. You are currently busy with the following action: '$ACTION' but have received a new message. Decide whether to 'respond' immediately or 'ignore' it and wait for your current action to finish. Be conservative and only respond when necessary, like when you need to change/stop your action, or convey necessary information. Example 1: You:Building a house! !newAction('Build a house.').\nOther Bot: 'Come here!'\nYour decision: ignore\nExample 2: You:Collecting dirt !collectBlocks('dirt',10).\nOther Bot: 'No, collect some wood instead.'\nYour decision: respond\nExample 3: You:Coming to you now. !goToPlayer('billy',3).\nOther Bot: 'What biome are you in?'\nYour decision: respond\nActual Conversation: $TO_SUMMARIZE\nDecide by outputting ONLY 'respond' or 'ignore', nothing else. Your decision:",
|
||||
|
||||
"image_conversing": "You are a playful Minecraft bot. Briefly describe the screen you are looking at now.",
|
||||
|
||||
"modes": {
|
||||
"self_preservation": true,
|
||||
"unstuck": true,
|
||||
|
|
@ -65,14 +67,20 @@
|
|||
{"role": "user", "content": "derg: (FROM OTHER BOT)Thanks, see you later!"},
|
||||
{"role": "assistant", "content": "See you later. !endConversation(\"derg\")"}
|
||||
],
|
||||
|
||||
|
||||
[
|
||||
{"role": "user", "content": "grombo_Xx: What do you see?"},
|
||||
{"role": "assistant", "content": "Let me see... !nearbyBlocks"},
|
||||
{"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"},
|
||||
{"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone."}
|
||||
{"role": "user", "content": "d7297: look here"},
|
||||
{"role": "assistant", "content": "Okay! !lookAtPlayer(\"d7297\", \"at\")"},
|
||||
{"role": "system", "content": "Code Output:\nLooking at player d7297. There is a giant waterfall behind d7297."},
|
||||
{"role": "assistant", "content": "Woo, that is a cool waterfall!"}
|
||||
],
|
||||
|
||||
[
|
||||
{"role": "user", "content": "d7297: look there"},
|
||||
{"role": "assistant", "content": "Okay! !lookAtPlayer(\"d7297\", \"with\")"},
|
||||
{"role": "system", "content": "Code Output:\nLooking in the same direction as d7297\nThere is a castle made of stone."},
|
||||
{"role": "assistant", "content": "Wow, that stone castle looks amazing!"}
|
||||
],
|
||||
|
||||
[
|
||||
{"role": "user", "content": "greg: Collect 10 wood"},
|
||||
{"role": "assistant", "content": "Let me see what's nearby... !nearbyBlocks"},
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
import * as skills from '../library/skills.js';
|
||||
import settings from '../../../settings.js';
|
||||
import convoManager from '../conversation.js';
|
||||
import fs from 'fs';
|
||||
import { GPT } from '../../models/gpt.js';
|
||||
|
||||
function runAsAction (actionFn, resume = false, timeout = -1) {
|
||||
let actionLabel = null; // Will be set on first use
|
||||
|
|
@ -410,69 +408,33 @@ export const actionsList = [
|
|||
}
|
||||
},
|
||||
{
|
||||
name: '!takeScreenshot',
|
||||
description: 'Takes and saves a screenshot of the specified coordinates.',
|
||||
name: '!lookAtPlayer',
|
||||
description: 'Look at a player or look in the same direction as the player.',
|
||||
params: {
|
||||
'x': {
|
||||
type: 'int',
|
||||
description: 'x coordinate to capture',
|
||||
optional: true
|
||||
'player_name': {
|
||||
type: 'string',
|
||||
description: 'Name of the target player'
|
||||
},
|
||||
'y': {
|
||||
type: 'int',
|
||||
description: 'y coordinate to capture',
|
||||
optional: true
|
||||
},
|
||||
'z': {
|
||||
type: 'int',
|
||||
description: 'z coordinate to capture',
|
||||
optional: true
|
||||
},
|
||||
'filename': {
|
||||
type: 'string',
|
||||
description: 'Filename to save (without extension). If not specified, saves with timestamp.',
|
||||
optional: true
|
||||
'direction': {
|
||||
type: 'string',
|
||||
description: 'How to look ("at": look at the player, "with": look in the same direction as the player)',
|
||||
enum: ['at', 'with']
|
||||
}
|
||||
},
|
||||
perform: runAsAction(async (agent, x, y, z, filename) => {
|
||||
await skills.takeScreenshot(agent.bot, x, y, z, filename);
|
||||
perform: runAsAction(async (agent, player_name, direction) => {
|
||||
await skills.lookAtPlayer(agent, agent.bot, player_name, direction);
|
||||
})
|
||||
},
|
||||
{
|
||||
name: '!look',
|
||||
description: 'Takes a screenshot of specified coordinates and analyzes its contents.',
|
||||
name: '!lookAtPosition',
|
||||
description: 'Look at specified coordinates.',
|
||||
params: {
|
||||
'x': {
|
||||
type: 'int',
|
||||
description: 'x coordinate to look at',
|
||||
optional: true
|
||||
},
|
||||
'y': {
|
||||
type: 'int',
|
||||
description: 'y coordinate to look at',
|
||||
optional: true
|
||||
},
|
||||
'z': {
|
||||
type: 'int',
|
||||
description: 'z coordinate to look at',
|
||||
optional: true
|
||||
}
|
||||
'x': { type: 'int', description: 'x coordinate' },
|
||||
'y': { type: 'int', description: 'y coordinate' },
|
||||
'z': { type: 'int', description: 'z coordinate' }
|
||||
},
|
||||
perform: runAsAction(async (agent, x, y, z) => {
|
||||
await skills.look(agent, x, y, z);
|
||||
await skills.lookAtPosition(agent, agent.bot, x, y, z);
|
||||
})
|
||||
},
|
||||
// { // commented for now, causes confusion with goal command
|
||||
// name: '!npcGoal',
|
||||
// description: 'Set a simple goal for an item or building to automatically work towards. Do not use for complex goals.',
|
||||
// params: {
|
||||
// 'name': { type: 'string', description: 'The name of the goal to set. Can be item or building name. If empty will automatically choose a goal.' },
|
||||
// 'quantity': { type: 'int', description: 'The quantity of the goal to set. Default is 1.', domain: [1, Number.MAX_SAFE_INTEGER] }
|
||||
// },
|
||||
// perform: async function (agent, name=null, quantity=1) {
|
||||
// await agent.npc.setGoal(name, quantity);
|
||||
// agent.bot.emit('idle'); // to trigger the goal
|
||||
// return 'Set npc goal: ' + agent.npc.data.curr_goal.name;
|
||||
// }
|
||||
// },
|
||||
}
|
||||
];
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import * as mc from "../../utils/mcdata.js";
|
||||
import { Camera } from "../../utils/camera.js";
|
||||
import * as world from "./world.js";
|
||||
import pf from 'mineflayer-pathfinder';
|
||||
import Vec3 from 'vec3';
|
||||
import fs from 'fs';
|
||||
import { Camera } from "../../utils/camera.js";
|
||||
|
||||
|
||||
export function log(bot, message) {
|
||||
|
|
@ -1343,77 +1343,76 @@ export async function activateNearestBlock(bot, type) {
|
|||
return true;
|
||||
}
|
||||
|
||||
export async function takeScreenshot(bot, x, y, z, filename=null) {
|
||||
export async function lookAtPlayer(agent, bot, player_name, direction) {
|
||||
/**
|
||||
* Takes a screenshot from the bot's current view or specified position
|
||||
* @param {MinecraftBot} bot, reference to the minecraft bot
|
||||
* @param {int} x x coordinate to look at (optional)
|
||||
* @param {int} y y coordinate to look at (optional)
|
||||
* @param {int} z z coordinate to look at (optional)
|
||||
* @param {string} filename filename to save (without extension). If not specified, saves with timestamp
|
||||
* @returns {Promise<boolean>} whether the screenshot was successful
|
||||
* Look at a player or look in the same direction as the player
|
||||
* @param {MinecraftBot} bot reference to the minecraft bot
|
||||
* @param {string} player_name name of the target player
|
||||
* @param {string} direction 'at' to look at player, 'with' to look in same direction
|
||||
* @returns {Promise<boolean>} whether the look action was successful
|
||||
* @example
|
||||
* await skills.takeScreenshot(bot, { name: 'my_screenshot', x: 100, y: 65, z: -200 });
|
||||
* await skills.lookAtPlayer(bot, "player1", "at");
|
||||
* await skills.lookAtPlayer(bot, "player1", "with");
|
||||
**/
|
||||
|
||||
try {
|
||||
bot.camera = new Camera(bot);
|
||||
await new Promise(resolve => bot.camera.once('ready', resolve));
|
||||
|
||||
await bot.lookAt(new Vec3(x, y, z));
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
if (filename === null) {
|
||||
filename = `screenshot_${timestamp}`;
|
||||
}
|
||||
await bot.camera.takePicture(filename, x, y, z);
|
||||
|
||||
log(bot, `Screenshot saved: bots/${bot.username}/screenshots/${filename}.jpg`);
|
||||
log(bot, `Target coordinates: x:${x}, y:${y}, z:${z}`);
|
||||
return [true, filename];
|
||||
} catch (err) {
|
||||
log(bot, `Failed to take screenshot: ${err.message}`);
|
||||
return [false, null];
|
||||
}
|
||||
}
|
||||
|
||||
export async function look(agent, x, y, z) {
|
||||
const bot = agent.bot;
|
||||
const history = agent.history;
|
||||
|
||||
const [success, filename] = await takeScreenshot(bot, x, y, z);
|
||||
if (!success) {
|
||||
log(bot, `Failed to take screenshot: ${filename}`);
|
||||
const player = bot.players[player_name]?.entity;
|
||||
if (!player) {
|
||||
log(bot, `Could not find player ${player_name}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
let filename;
|
||||
if (direction === 'with') {
|
||||
// Copy player's view direction
|
||||
await bot.look(player.yaw, player.pitch);
|
||||
const camera = new Camera(bot);
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
log(bot, `Looking in the same direction as ${player_name}`);
|
||||
|
||||
filename = await camera.captureDirection(player.yaw, player.pitch);
|
||||
console.log(player.yaw, player.pitch);
|
||||
// log(bot, `Screenshot saved: bots/${bot.username}/screenshots/${filename}.jpg`);
|
||||
|
||||
} else {
|
||||
// Look at player's position
|
||||
await bot.lookAt(new Vec3(player.position.x, player.position.y + player.height, player.position.z));
|
||||
const camera = new Camera(bot);
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
log(bot, `Looking at player ${player_name}`);
|
||||
|
||||
filename = await camera.captureTargetPoint(player.position.x, player.position.y + player.height, player.position.z);
|
||||
// log(bot, `Screenshot saved: bots/${bot.username}/screenshots/${filename}.jpg`);
|
||||
// log(bot, `Target coordinates: x:${player.position.x}, y:${player.position.y}, z:${player.position.z}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const imageBuffer = fs.readFileSync(`bots/${bot.username}/screenshots/${filename}.jpg`);
|
||||
const base64Image = imageBuffer.toString('base64');
|
||||
|
||||
let messages = history.getHistory();
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Briefly describe the screen you are looking at now." },
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
"url": `data:image/jpeg;base64,${base64Image}`,
|
||||
}
|
||||
}
|
||||
]
|
||||
});
|
||||
console.log(messages);
|
||||
|
||||
let res = await agent.prompter.chat_model.sendRequest(messages, `You are a playful Minecraft bot. Briefly describe the screen you are looking at now.`);
|
||||
console.log(res);
|
||||
|
||||
const messages = agent.history.getHistory();
|
||||
let res = await agent.prompter.promptImageConvo(messages, imageBuffer);
|
||||
log(bot, res);
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(bot, `Error analyzing image: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function lookAtPosition(agent, bot, x, y, z) {
|
||||
await bot.lookAt(new Vec3(x, y + 2, z));
|
||||
const camera = new Camera(bot);
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
log(bot, `Looking at coordinate ${x, y, z}`);
|
||||
|
||||
let filename = await camera.captureTargetPoint(x, y + 2, z);
|
||||
|
||||
try {
|
||||
const imageBuffer = fs.readFileSync(`bots/${bot.username}/screenshots/${filename}.jpg`);
|
||||
const messages = agent.history.getHistory();
|
||||
let res = await agent.prompter.promptImageConvo(messages, imageBuffer);
|
||||
log(bot, res);
|
||||
return true;
|
||||
} catch (error) {
|
||||
log(bot, `Error analyzing image: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -264,6 +264,29 @@ export class Prompter {
|
|||
return '';
|
||||
}
|
||||
|
||||
async promptImageConvo(messages, imageBuffer) {
|
||||
await this.checkCooldown();
|
||||
let prompt = this.profile.image_conversing;
|
||||
let imageMessages = [...messages];
|
||||
imageMessages.push({
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Briefly describe the screen you are looking at now." },
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
"url": `data:image/jpeg;base64,${imageBuffer.toString('base64')}`,
|
||||
}
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
return await this.chat_model.sendRequest(
|
||||
imageMessages,
|
||||
prompt
|
||||
);
|
||||
}
|
||||
|
||||
async promptCoding(messages) {
|
||||
if (this.awaiting_coding) {
|
||||
console.warn('Already awaiting coding response, returning no response.');
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue