diff --git a/profiles/_default.json b/profiles/_default.json index 90a1621..f0ed592 100644 --- a/profiles/_default.json +++ b/profiles/_default.json @@ -80,6 +80,12 @@ {"role": "system", "content": "Code Output:\nLooking in the same direction as d7297\nThere is a castle made of stone."}, {"role": "assistant", "content": "Wow, that stone castle looks amazing!"} ], + [ + {"role": "user", "content": "d7297: look 0 70 12 and describe what is there"}, + {"role": "assistant", "content": "Sure! !lookAtPosition(0, 70, 12)"}, + {"role": "system", "content": "Code Output:\nLooking in the same direction as d7297\nThere is a water fountain."}, + {"role": "assistant", "content": "I see the water fountain! amazing!"} + ], [ {"role": "user", "content": "greg: Collect 10 wood"}, diff --git a/src/agent/prompter.js b/src/agent/prompter.js index 11ae554..4c7507d 100644 --- a/src/agent/prompter.js +++ b/src/agent/prompter.js @@ -287,6 +287,29 @@ export class Prompter { ); } + async promptImageConvo(messages, imageBuffer) { + await this.checkCooldown(); + let prompt = this.profile.image_conversing; + let imageMessages = [...messages]; + imageMessages.push({ + role: "user", + content: [ + { type: "text", text: "Briefly describe the screen you are looking at now." }, + { + type: "image_url", + image_url: { + "url": `data:image/jpeg;base64,${imageBuffer.toString('base64')}`, + } + } + ] + }); + + return await this.chat_model.sendRequest( + imageMessages, + prompt + ); + } + async promptCoding(messages) { if (this.awaiting_coding) { console.warn('Already awaiting coding response, returning no response.'); diff --git a/src/utils/camera.js b/src/utils/camera.js new file mode 100644 index 0000000..54a0c7e --- /dev/null +++ b/src/utils/camera.js @@ -0,0 +1,88 @@ +import { Viewer } from 'prismarine-viewer/viewer/lib/viewer.js'; +import { WorldView } from 'prismarine-viewer/viewer/lib/worldview.js'; +import { getBufferFromStream } from 'prismarine-viewer/viewer/lib/simpleUtils.js'; + +import THREE from 'three'; +import { createCanvas } from 'node-canvas-webgl/lib/index.js'; +import fs from 'fs/promises'; +import { Vec3 } from 'vec3'; +import { EventEmitter } from 'events'; + +import worker_threads from 'worker_threads'; +global.Worker = worker_threads.Worker; + + +export class Camera extends EventEmitter { + constructor (bot) { + super() + this.bot = bot + this.viewDistance = 4 + this.width = 800 + this.height = 512 + this.canvas = createCanvas(this.width, this.height) + this.renderer = new THREE.WebGLRenderer({ canvas: this.canvas }) + this.viewer = new Viewer(this.renderer) + this._init().then(() => { + this.emit('ready') + }) + } + + async _init () { + const botPos = this.bot.entity.position + const center = new Vec3(botPos.x, botPos.y+this.bot.entity.height, botPos.z) + this.viewer.setVersion(this.bot.version) + // Load world + const worldView = new WorldView(this.bot.world, this.viewDistance, center) + this.viewer.listen(worldView) + + this.viewer.camera.position.set(center.x, center.y, center.z) + + await worldView.init(center) + } + + async captureTargetPoint(x, y, z) { + this.viewer.camera.lookAt(x, y, z); + const filename = await this._capture(); + return filename; + } + + async captureDirection(yaw, pitch) { + this.viewer.camera.rotation.y = yaw; + this.viewer.camera.rotation.x = pitch; + const filename = await this._capture(); + return filename; + } + + async _capture() { + console.info('Waiting for camera to load'); + await new Promise(resolve => setTimeout(resolve, 5000)); + this.renderer.render(this.viewer.scene, this.viewer.camera); + + const imageStream = this.canvas.createJPEGStream({ + bufsize: 4096, + quality: 100, + progressive: false + }); + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `screenshot_${timestamp}`; + + const buf = await getBufferFromStream(imageStream); + await this._ensureScreenshotDirectory(); + await fs.writeFile(`bots/${this.bot.username}/screenshots/${filename}.jpg`, buf); + console.log('saved', filename); + return filename; + } + + async _ensureScreenshotDirectory() { + let stats; + try { + stats = await fs.stat(`bots/${this.bot.username}/screenshots`); + } catch (e) { + if (!stats?.isDirectory()) { + await fs.mkdir(`bots/${this.bot.username}/screenshots`); + } + } + } +} + \ No newline at end of file