From 86ab1686b403637133f10ac54d3404a09805a1ad Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Tue, 30 Jan 2024 16:43:30 -0600 Subject: [PATCH 1/6] added settings, fixed biome name --- README.md | 31 ++++++++++++++++++++++++++----- settings.json | 6 ++++++ src/agent/agent.js | 4 +++- src/agent/commands/actions.js | 4 +++- src/agent/library/world.js | 4 ++-- src/examples_coder.json | 2 +- src/settings.js | 3 +++ src/utils/gpt.js | 8 +++++++- src/utils/mcdata.js | 16 +++++++++------- 9 files changed, 60 insertions(+), 18 deletions(-) create mode 100644 settings.json create mode 100644 src/settings.js diff --git a/README.md b/README.md index 7bd3dd5..941b8fa 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,40 @@ Crafting minds for Minecraft with AI! +### ‼️Warning‼️ + +This project allows an AI model to write/execute code on your computer that may be insecure, dangerous, and vulnerable to injection attacks by human players. This is disabled by default, you can enable it by setting `allow_insecure_coding` to `true` in `settings.json`. Use with caution. + + +**Do not** connect this bot to public servers, only run on local or private servers. + ## Installation +Install Minecraft Java Edition <= 1.20.2 + Install Node.js >= 14 from [nodejs.org](https://nodejs.org/) -Install node modules with `npm install` +Clone/Download this repository -## Usage - -Start minecraft server on localhost port `55916` +Run `npm install` Add `OPENAI_API_KEY` (and optionally `OPENAI_ORG_ID`) to your environment variables. -run `node main.js` +## Running + +Start minecraft game and open it to LAN on localhost port `55916` + +Run `node main.js` + +You can configure the bot in `settings.json`. Here is an example settings for connecting to a non-local server: +``` +{ + "host": "111.222.333.444", + "port": 55920, + "auth": "microsoft", + "allow_insecure_coding": false +} +``` ## Patches diff --git a/settings.json b/settings.json new file mode 100644 index 0000000..854c012 --- /dev/null +++ b/settings.json @@ -0,0 +1,6 @@ +{ + "host": "localhost", + "port": 55916, + "auth": "offline", + "allow_insecure_coding": false +} \ No newline at end of file diff --git a/src/agent/agent.js b/src/agent/agent.js index 1ca99a6..5466ea4 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -14,16 +14,18 @@ export class Agent { this.history = new History(this); this.coder = new Coder(this); + console.log('Loading examples...'); + this.history.load(profile); await this.examples.load('./src/examples.json'); await this.coder.load(); + console.log('Logging in...'); this.bot = initBot(name); initModes(this); this.bot.on('login', async () => { - console.log(`${this.name} logged in.`); this.coder.clear(); diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index 4596b04..21150db 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -1,5 +1,5 @@ import * as skills from '../library/skills.js'; - +import settings from '../../settings.js'; function wrapExecution(func, timeout=-1) { return async function (agent, ...args) { @@ -17,6 +17,8 @@ export const actionsList = [ name: '!newAction', description: 'Perform new and unknown custom behaviors that are not available as a command by writing code.', perform: async function (agent) { + if (!settings.allow_insecure_coding) + return 'Agent is not allowed to write code.'; await agent.coder.generateCode(agent.history); } }, diff --git a/src/agent/library/world.js b/src/agent/library/world.js index 52c4b74..50cdfcc 100644 --- a/src/agent/library/world.js +++ b/src/agent/library/world.js @@ -1,5 +1,5 @@ import pf from 'mineflayer-pathfinder'; -import { getAllBlockIds } from '../../utils/mcdata.js'; +import { getAllBlockIds, getAllBiomes } from '../../utils/mcdata.js'; export function getNearestFreeSpace(bot, size=1, distance=8) { @@ -271,5 +271,5 @@ export function getBiomeName(bot) { * let biome = world.getBiomeName(bot); **/ const biomeId = bot.world.getBiome(bot.entity.position); - return mcdata.biomes[biomeId].name; + return getAllBiomes()[biomeId].name; } diff --git a/src/examples_coder.json b/src/examples_coder.json index 9e20073..9ec7435 100644 --- a/src/examples_coder.json +++ b/src/examples_coder.json @@ -33,6 +33,6 @@ ], [ {"role": "user", "content": "brug: build a dirt house"}, - {"role": "assistant", "content": "```const position = world.getPosition(bot);\nconst startX = position.x;\nconst startY = position.y;\nconst startZ = position.z;\nconst width = 7;\nconst depth = 7;\nconst height = 4;\n\n// Build the walls\nfor (let x = startX; x < startX + width; x++) {\n for (let y = startY; y < startY + height; y++) {\n for (let z = startZ; z < startZ + depth; z++) {\n if (x === startX || x === startX + width - 1 || y === startY || y === startY + height - 1 || z === startZ || z === startZ + depth - 1) {\n await skills.placeBlock(bot, 'oak_planks', x, y, z); \n }\n }\n }\n}\n// place a torch in the middle of the room\nawait skills.placeBlock(bot, 'torch', startX + width / 2, startY + height / 2, startZ + 1);```"} + {"role": "assistant", "content": "```const position = world.getPosition(bot);\nconst startX = position.x;\nconst startY = position.y;\nconst startZ = position.z;\nconst width = 7;\nconst depth = 7;\nconst height = 4;\n\n// Build the walls\nfor (let x = startX; x < startX + width; x++) {\n for (let y = startY; y < startY + height; y++) {\n for (let z = startZ; z < startZ + depth; z++) {\n if (x === startX || x === startX + width - 1 || y === startY || y === startY + height - 1 || z === startZ || z === startZ + depth - 1) {\n await skills.placeBlock(bot, 'oak_planks', x, y, z); \n }\n }\n }\n}\n```"} ] ] \ No newline at end of file diff --git a/src/settings.js b/src/settings.js new file mode 100644 index 0000000..00976c5 --- /dev/null +++ b/src/settings.js @@ -0,0 +1,3 @@ +import { readFileSync } from 'fs'; +const settings = JSON.parse(readFileSync('./settings.json', 'utf8')); +export default settings; \ No newline at end of file diff --git a/src/utils/gpt.js b/src/utils/gpt.js index 0780103..b5281c7 100644 --- a/src/utils/gpt.js +++ b/src/utils/gpt.js @@ -7,11 +7,17 @@ if (process.env.OPENAI_ORG_ID) { organization: process.env.OPENAI_ORG_ID, apiKey: process.env.OPENAI_API_KEY, }; -} else { +} +else if (process.env.OPENAI_API_KEY) { openAiConfig = { apiKey: process.env.OPENAI_API_KEY, }; } +else { + console.error('OpenAI API key missing! Make sure you set OPENAI_API_KEY and OPENAI_ORG_ID (optional) environment variables.'); + process.exit(1); +} + const openai = new OpenAIApi(openAiConfig); diff --git a/src/utils/mcdata.js b/src/utils/mcdata.js index dcd80e5..7c8a0a9 100644 --- a/src/utils/mcdata.js +++ b/src/utils/mcdata.js @@ -1,4 +1,5 @@ import minecraftData from 'minecraft-data'; +import settings from '../settings.js'; import { createBot } from 'mineflayer'; import { pathfinder } from 'mineflayer-pathfinder'; import { plugin as pvp } from 'mineflayer-pvp'; @@ -7,7 +8,7 @@ import { plugin as autoEat } from 'mineflayer-auto-eat'; import plugin from 'mineflayer-armor-manager'; const armorManager = plugin; -const mc_version = '1.20.1' +const mc_version = settings.minecraft_version; const mcdata = minecraftData(mc_version); @@ -15,12 +16,9 @@ export function initBot(username) { let bot = createBot({ username: username, - host: 'localhost', - port: 55916, - - // host: '000.111.222.333', - // port: 55920, - // auth: 'microsoft', + host: settings.host, + port: settings.port, + auth: settings.auth, version: mc_version, }); @@ -97,3 +95,7 @@ export function getAllBlockIds(ignore) { } return blockIds; } + +export function getAllBiomes() { + return mcdata.biomes; +} \ No newline at end of file From 74e061c48f58ad56b44f22826785df36b6e77091 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Tue, 30 Jan 2024 22:26:27 -0600 Subject: [PATCH 2/6] improved readme --- README.md | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 941b8fa..3d8175c 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,32 @@ # Mindcraft -Crafting minds for Minecraft with AI! +Crafting minds for Minecraft with ChatGPT and Mineflayer -### ‼️Warning‼️ +#### ‼️Warning‼️ -This project allows an AI model to write/execute code on your computer that may be insecure, dangerous, and vulnerable to injection attacks by human players. This is disabled by default, you can enable it by setting `allow_insecure_coding` to `true` in `settings.json`. Use with caution. +This project allows an AI model to write/execute code on your computer that may be insecure, dangerous, and vulnerable to injection attacks on public servers. Code writing is disabled by default, you can enable it by setting `allow_insecure_coding` to `true` in `settings.json`. Enable only on local or private servers, **never** on public servers. Ye be warned. +## Requirements -**Do not** connect this bot to public servers, only run on local or private servers. +- [OpenAI API Subscription](https://openai.com/blog/openai-api) +- [Minecraft Java Edition](https://www.minecraft.net/en-us/store/minecraft-java-bedrock-edition-pc) (at most v1.20.2) +- [Node.js](https://nodejs.org/) (at least v14) ## Installation -Install Minecraft Java Edition <= 1.20.2 - -Install Node.js >= 14 from [nodejs.org](https://nodejs.org/) +Add `OPENAI_API_KEY` (and optionally `OPENAI_ORG_ID`) to your environment variables Clone/Download this repository Run `npm install` -Add `OPENAI_API_KEY` (and optionally `OPENAI_ORG_ID`) to your environment variables. +## Run -## Running - -Start minecraft game and open it to LAN on localhost port `55916` +Start a minecraft world and open it to LAN on localhost port `55916` Run `node main.js` -You can configure the bot in `settings.json`. Here is an example settings for connecting to a non-local server: +You can configure details in `settings.json`. Here is an example settings for connecting to a non-local server: ``` { "host": "111.222.333.444", From ab340a7b6d02b87598557be013205af090f240f0 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Thu, 1 Feb 2024 16:16:30 -0600 Subject: [PATCH 3/6] small fixes --- README.md | 1 + settings.json | 3 ++- src/agent/library/skills.js | 37 ++++++++++++++++++++----------------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3d8175c..63477ff 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Run `node main.js` You can configure details in `settings.json`. Here is an example settings for connecting to a non-local server: ``` { + "minecraft_version": "1.20.1", "host": "111.222.333.444", "port": 55920, "auth": "microsoft", diff --git a/settings.json b/settings.json index 854c012..8af8c75 100644 --- a/settings.json +++ b/settings.json @@ -1,6 +1,7 @@ { + "minecraft_version": "1.20.1", "host": "localhost", "port": 55916, "auth": "offline", - "allow_insecure_coding": false + "allow_insecure_coding": true } \ No newline at end of file diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index d2a76e9..ee6c734 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -278,7 +278,7 @@ export async function attackEntity(bot, entity, kill=true) { } } -export async function defendSelf(bot, range=8) { +export async function defendSelf(bot, range=9) { /** * Defend yourself from all nearby hostile mobs until there are no more. * @param {MinecraftBot} bot, reference to the minecraft bot. @@ -336,24 +336,27 @@ export async function collectBlock(bot, blockType, num=1) { blocktypes.push('deepslate_'+blockType); let collected = 0; - const blocks = world.getNearestBlocks(bot, blocktypes, 64, num); - if (blocks.length === 0) { - log(bot, `Could not find any ${blockType} to collect.`); - return false; - } - const first_block = blocks[0]; - await bot.tool.equipForBlock(first_block); - const itemId = bot.heldItem ? bot.heldItem.type : null - if (!first_block.canHarvest(itemId)) { - log(bot, `Don't have right tools to harvest ${blockType}.`); - return false; - } - for (let block of blocks) { + for (let i=0; i mc.isHostile(entity), attack_distance); if (enemy) { log(bot, `Found ${enemy.name}, attacking!`, true); - await defendSelf(bot, 8); + await defendSelf(bot); acted = true; } } @@ -680,7 +683,7 @@ export async function followPlayer(bot, username) { } } if (bot.entity.position.distanceTo(player.position) < follow_distance) { - acted = autoLight(bot); + acted = await autoLight(bot); } if (acted) { // if we did something then resume following From 578fcf99ba851e09de8898891c63717a384957be Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Fri, 2 Feb 2024 11:54:17 -0600 Subject: [PATCH 4/6] added moveAway and stay commands --- src/agent/commands/actions.js | 15 ++++++++++++++ src/agent/library/skills.js | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index 21150db..94e5114 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -79,6 +79,14 @@ export const actionsList = [ await skills.followPlayer(agent.bot, player_name); }) }, + { + name: '!moveAway', + description: 'Move away from the current location in any direction by a given distance. Ex: !moveAway(2)', + params: {'distance': '(number) The distance to move away.'}, + perform: wrapExecution(async (agent, distance) => { + await skills.moveAway(agent.bot, distance); + }) + }, { name: '!givePlayer', description: 'Give the specified item to the given player. Ex: !givePlayer("steve", "stone_pickaxe", 1)', @@ -138,5 +146,12 @@ export const actionsList = [ perform: wrapExecution(async (agent) => { await skills.goToBed(agent.bot); }) + }, + { + name: '!stay', + description: 'Stay in the current location no matter what. Pauses all modes.', + perform: wrapExecution(async (agent) => { + await skills.stay(agent.bot); + }) } ]; diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index ee6c734..0bf3d0b 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -697,6 +697,44 @@ export async function followPlayer(bot, username) { } +export async function moveAway(bot, distance) { + /** + * Move away from current position in any direction. + * @param {MinecraftBot} bot, reference to the minecraft bot. + * @param {number} distance, the distance to move away. + * @returns {Promise} true if the bot moved away, false otherwise. + * @example + * await skills.moveAway(bot, 8); + **/ + const pos = bot.entity.position; + let goal = new pf.goals.GoalNear(pos.x, pos.y, pos.z, distance); + let inverted_goal = new pf.goals.GoalInvert(goal); + bot.pathfinder.setMovements(new pf.Movements(bot)); + await bot.pathfinder.goto(inverted_goal); + let new_pos = bot.entity.position; + log(bot, `Moved away from nearest entity to ${new_pos}.`); + return true; +} + +export async function stay(bot) { + /** + * Stay in the current position until interrupted. Disables all modes. + * @param {MinecraftBot} bot, reference to the minecraft bot. + * @returns {Promise} true if the bot stayed, false otherwise. + * @example + * await skills.stay(bot); + **/ + bot.modes.pause('self_defense'); + bot.modes.pause('hunting'); + bot.modes.pause('torch_placing'); + bot.modes.pause('item_collecting'); + while (!bot.interrupt_code) { + await new Promise(resolve => setTimeout(resolve, 500)); + } + return true; +} + + export async function goToBed(bot) { /** * Sleep in the nearest bed. From f203b7ca4e635374a56c66b7ab8f1b50906d38f9 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Fri, 2 Feb 2024 15:47:17 -0600 Subject: [PATCH 5/6] made item collecting mode better --- src/agent/library/skills.js | 30 ++++++++++++++++++------------ src/agent/modes.js | 26 ++++++++++++++++++++------ 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index 0bf3d0b..5fb46f1 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -273,7 +273,7 @@ export async function attackEntity(bot, entity, kill=true) { } } log(bot, `Successfully killed ${entity.name}.`); - await pickupNearbyItem(bot); + await pickupNearbyItems(bot); return true; } } @@ -376,24 +376,30 @@ export async function collectBlock(bot, blockType, num=1) { return true; } -export async function pickupNearbyItem(bot) { +export async function pickupNearbyItems(bot) { /** * Pick up all nearby items. * @param {MinecraftBot} bot, reference to the minecraft bot. * @returns {Promise} true if the items were picked up, false otherwise. * @example - * await skills.pickupNearbyItem(bot); + * await skills.pickupNearbyItems(bot); **/ - const distance = 10; - let nearestItem = bot.nearestEntity(entity => entity.name === 'item' && bot.entity.position.distanceTo(entity.position) < distance); - - if (!nearestItem) { - log(bot, `Didn't pick up items.`); - return false; + const distance = 8; + const getNearestItem = bot => bot.nearestEntity(entity => entity.name === 'item' && bot.entity.position.distanceTo(entity.position) < distance); + let nearestItem = getNearestItem(bot); + let pickedUp = 0; + while (nearestItem) { + bot.pathfinder.setMovements(new pf.Movements(bot)); + await bot.pathfinder.goto(new pf.goals.GoalFollow(nearestItem, 0.8), true); + await new Promise(resolve => setTimeout(resolve, 200)); + let prev = nearestItem; + nearestItem = getNearestItem(bot); + if (prev === nearestItem) { + break; + } + pickedUp++; } - bot.pathfinder.setMovements(new pf.Movements(bot)); - await bot.pathfinder.goto(new pf.goals.GoalFollow(nearestItem, 0.8), true); - log(bot, `Successfully picked up a dropped item.`); + log(bot, `Picked up ${pickedUp} items.`); return true; } diff --git a/src/agent/modes.js b/src/agent/modes.js index 20ae23c..1639d92 100644 --- a/src/agent/modes.js +++ b/src/agent/modes.js @@ -53,15 +53,29 @@ const modes = [ description: 'Automatically collect nearby items when idle.', on: true, active: false, + + wait: 2, // number of seconds to wait after noticing an item to pick it up + prev_item: null, + noticed_at: -1, update: async function (agent) { + if (this.active) return; if (agent.isIdle()) { let item = world.getNearestEntityWhere(agent.bot, entity => entity.name === 'item', 8); - if (item && await world.isClearPath(agent.bot, item)) { - execute(this, agent, async () => { - // wait 2 seconds for the item to settle - await new Promise(resolve => setTimeout(resolve, 2000)); - await skills.pickupNearbyItem(agent.bot); - }); + if (item && item !== this.prev_item && await world.isClearPath(agent.bot, item)) { + if (this.noticed_at === -1) { + this.noticed_at = Date.now(); + } + if (Date.now() - this.noticed_at > this.wait * 1000) { + agent.bot.chat(`Picking up ${item.name}!`); + this.prev_item = item; + execute(this, agent, async () => { + await skills.pickupNearbyItems(agent.bot); + }); + this.noticed_at = -1; + } + } + else { + this.noticed_at = -1; } } } From 913b39b0b6195b88290673a97ea2b67c260ef4c3 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Sat, 3 Feb 2024 12:00:33 -0600 Subject: [PATCH 6/6] crash when fail to stop, improve digging --- src/agent/coder.js | 6 +++++- src/agent/library/skills.js | 21 ++++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/agent/coder.js b/src/agent/coder.js index d4d842a..f564abb 100644 --- a/src/agent/coder.js +++ b/src/agent/coder.js @@ -210,13 +210,17 @@ export class Coder { async stop() { if (!this.executing) return; + const start = Date.now(); while (this.executing) { this.agent.bot.interrupt_code = true; this.agent.bot.collectBlock.cancelTask(); this.agent.bot.pathfinder.stop(); this.agent.bot.pvp.stop(); - console.log('waiting for code to finish executing... interrupt:', this.agent.bot.interrupt_code); + console.log('waiting for code to finish executing...'); await new Promise(resolve => setTimeout(resolve, 1000)); + if (Date.now() - start > 10 * 1000) { + process.exit(1); // force exit program after 10 seconds of failing to stop + } } } diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index 5fb46f1..906b24b 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -416,9 +416,24 @@ export async function breakBlockAt(bot, x, y, z) { * let position = world.getPosition(bot); * await skills.breakBlockAt(bot, position.x, position.y - 1, position.x); **/ - let current = bot.blockAt(Vec3(x, y, z)); - if (current.name != 'air') - await bot.dig(current, true); + let block = bot.blockAt(Vec3(x, y, z)); + if (block.name !== 'air' && block.name !== 'water' && block.name !== 'lava') { + await bot.tool.equipForBlock(block); + const itemId = bot.heldItem ? bot.heldItem.type : null + if (!block.canHarvest(itemId)) { + log(bot, `Don't have right tools to break ${block.name}.`); + return false; + } + if (bot.entity.position.distanceTo(block.position) > 4.5) { + let pos = block.position; + let movements = new pf.Movements(bot); + movements.canPlaceOn = false; + movements.allow1by1towers = false; + bot.pathfinder.setMovements(); + await bot.pathfinder.goto(new pf.goals.GoalNear(pos.x, pos.y, pos.z, 4)); + } + await bot.dig(block, true); + } return true; }