init commit

.gitignore

@@ -0,0 +1,4 @@
+.vscode/
+node_modules/
+package-lock.json
+temp.js

README.md

@@ -0,0 +1,22 @@
+# Mindcraft
+
+Crafting minds for Minecraft with AI!
+
+## Installation
+
+Install Node.js >= 14 from [nodejs.org](https://nodejs.org/)
+
+Then, install mineflayer
+```
+npm install mineflayer
+npm install mineflayer-pathfinder
+npm install mineflayer-collectblock
+```
+
+## Usage
+
+Start minecraft server on localhost port `55916`
+
+Add `OPENAI_API_KEY` (and optionally `OPENAI_ORG_ID`) to your environment variables.
+
+run `npm main.py`

act.js

@@ -0,0 +1,100 @@
+import { writeFile } from 'fs';
+
+import { getStats, getInventory, getNearbyBlocks, getNearbyPlayers, getNearbyEntities, getCraftable, getDetailedSkills } from './utils/context.js';
+import { sendRequest } from './utils/gpt.js';
+
+
+function buildSystemMessage(bot) {
+    let message = 'You are a helpful Minecraft bot. Given the dialogue and currently running program, reflect on what you are doing and generate javascript code to accomplish that goal. If your new code is empty, no change will be made to your currently running program. Use only functions listed below to write your code.';
+    let stats = getStats(bot);
+    if (stats)
+        message += "\n\n" + stats;
+    let inventory = getInventory(bot);
+    if (inventory)
+        message += "\n\n" + inventory;
+    let nearbyBlocks = getNearbyBlocks(bot);
+    if (nearbyBlocks)
+        message += "\n\n" + nearbyBlocks;
+    let nearbyPlayers = getNearbyPlayers(bot);
+    if (nearbyPlayers)
+        message += "\n\n" + nearbyPlayers;
+    let nearbyEntities = getNearbyEntities(bot);
+    if (nearbyEntities)
+        message += "\n\n" + nearbyEntities;
+    let craftable = getCraftable(bot);
+    if (craftable)
+        message += "\n\n" + craftable;
+    let skills = getDetailedSkills();
+    if (skills)
+        message += "\n\n" + skills;
+    return message;
+}
+
+
+function buildExamples() {
+    return[
+`mr_steve2: Will you help me collect wood?
+You: I'd be glad to help you collect wood.
+Current code:
+\`\`\`
+await skills.ExploreToFind(bot, 'iron_ore');
+\`\`\``,
+`I'm going to help mr_steve2 collect wood rather than look for iron ore. The type of wood block nearby is 'oak_log'. I'll adjust my code to collect 'oak_log' for mr_steve2 until told to stop.
+\`\`\`
+while (true) {
+    await skills.CollectBlock(bot, 'oak_log', 1);
+    await skills.GoToPlayer(bot, 'mr_steve2');
+    await skills.DropItem(bot, 'oak_log', 1);
+}
+\`\`\``,
+`sally32: What are you doing?
+You: I'm looking for coal. Have you seen any?
+Current code:
+\`\`\`
+await skills.ExploreToFind(bot, 'coal_ore');
+await skills.EquipItem(bot, 'wooden_pickaxe');
+await skills.CollectBlock(bot, 'coal_ore', 10);
+\`\`\``,
+`I responded to a question. I do not need to change my code.
+\`\`\`
+\`\`\``,
+    ]
+}
+
+
+async function executeCode(bot, code) {
+    let src = `import * as skills from './utils/skills.js';\n\n`;
+    src += `export async function main(bot) {\n`;
+    for (let line of code.split('\n')) {
+        src += `    ${line}\n`;
+    }
+    src += `}\n`;
+    
+    writeFile('./temp.js', src, (err) => {
+        if (err) throw err;
+    });
+    await (await import('./temp.js')).main(bot);
+}
+
+
+var currentCode = '';
+export async function actIfNeeded(bot, username, message, res) {
+    let turns = buildExamples();
+    turns.push(`${username}: ${message}\nYou: ${res}\nCurrent Code:\`\`\`\n${currentCode}\n\`\`\``);
+    let systemMessage = buildSystemMessage(bot);
+    let actResponse = await sendRequest(turns, systemMessage);
+    console.log(actResponse);
+
+    let code = actResponse.split('\`\`\`');
+    if (code.length <= 1)
+        return false;
+    if (!code[1].trim())
+        return false;
+
+    currentCode = code[1].trim();
+    if (currentCode.slice(0, 10) == 'javascript')
+        currentCode = currentCode.slice(10).trim();
+
+    await executeCode(bot, currentCode);
+    return true;
+}

chat.js

@@ -0,0 +1,67 @@
+import { sendRequest } from './utils/gpt.js';
+import { getHistory, addEvent } from './utils/history.js';
+import { getStats, getInventory, getNearbyBlocks, getNearbyPlayers, getNearbyEntities, getCraftable } from './utils/context.js';
+
+
+function buildSystemMessage(bot) {
+    let message = 'You are a helpful Minecraft bot that can communicate with players and execute actions in the environment. Act human-like as if you were a typical Minecraft player, rather than an AI. Do not give instructions unless asked.';
+    let stats = getStats(bot);
+    if (stats)
+        message += "\n\n" + stats;
+    let inventory = getInventory(bot);
+    if (inventory)
+        message += "\n\n" + inventory;
+    let nearbyBlocks = getNearbyBlocks(bot);
+    if (nearbyBlocks)
+        message += "\n\n" + nearbyBlocks;
+    let nearbyPlayers = getNearbyPlayers(bot);
+    if (nearbyPlayers)
+        message += "\n\n" + nearbyPlayers;
+    let nearbyEntities = getNearbyEntities(bot);
+    if (nearbyEntities)
+        message += "\n\n" + nearbyEntities;
+    let craftable = getCraftable(bot);
+    if (craftable)
+        message += "\n\n" + craftable;
+    return message;
+}
+
+
+function buildTurns(user, message) {
+    let history = getHistory();
+
+    let turns = [];
+    let lastSource = null;
+    for (let i = 0; i < history.length; i++) {
+
+        if (history[i].source == 'bot' && lastSource == null) {
+            turns.push('(You spawn into the word.)');
+            turns.push(history[i].message);
+
+        } else if (history[i].source == 'bot' && lastSource != 'bot') {
+            turns.push(history[i].message);
+
+        } else if (history[i].source == 'bot' && lastSource == 'bot') {
+            turns[turns.length - 1] += '\n\n' + history[i].message;
+
+        } else if (history[i].source != 'bot' && lastSource == 'bot') {
+            turns.push(history[i].message);
+
+        } else if (history[i].source != 'bot' && lastSource != 'bot') {
+            turns[turns.length - 1] += '\n\n' + history[i].message;
+        }
+        lastSource = history[i].source;
+    }
+    return turns;
+}
+
+
+export async function getChatResponse(bot, user, message) {
+    addEvent(user, user + ': ' + message);
+    let turns = buildTurns(user, message);
+    let systemMessage = buildSystemMessage(bot);
+    let res = await sendRequest(turns, systemMessage);
+    console.log('sending chat:', res);
+    addEvent('bot', res);
+    return res;
+}

main.js

@@ -0,0 +1,36 @@
+import { createBot } from 'mineflayer';
+import { pathfinder } from 'mineflayer-pathfinder';
+import { plugin } from 'mineflayer-collectblock';
+
+import { getChatResponse } from './chat.js';
+import { actIfNeeded } from './act.js';
+
+
+async function handleMessage(username, message) {
+    if (username === bot.username) return;
+    console.log('received message from', username, ':', message);
+    
+    let chat = await getChatResponse(bot, username, message);
+    bot.chat(chat);
+
+    let actResult = await actIfNeeded(bot, username, message, chat);
+    if (actResult) {
+        console.log('completed action');
+    }
+}
+
+
+const bot = createBot({
+    host: '127.0.0.1',
+    port: 55916,
+    username: 'andy'
+})
+bot.loadPlugin(pathfinder)
+bot.loadPlugin(plugin)
+console.log('bot created')
+
+
+bot.on('chat', handleMessage);
+
+
+bot.on('whisper', handleMessage);

package.json

@@ -0,0 +1,8 @@
+{
+    "type": "module",
+    "dependencies": {
+        "mineflayer": "^4.11.0",
+        "mineflayer-collectblock": "^1.4.1",
+        "mineflayer-pathfinder": "^2.4.4"
+    }
+}

utils/context.js

@@ -0,0 +1,55 @@
+import { getDocstrings } from './skills.js';
+
+
+export function getStats(bot) {
+    return "";
+}
+
+
+export function getInventory(bot) {
+    return "";
+}
+
+
+export function getNearbyBlocks(bot) {
+    return "";
+}
+
+
+export function getNearbyEntities(bot) {
+    return "";
+}
+
+
+export function getNearbyPlayers(bot) {
+    return "";
+}
+
+
+export function getCraftable(bot) {
+    return "";
+}
+
+
+export function getSkills() {
+    let res = '';
+    let docs = getDocstrings();
+    let lines = null;
+    for (let i = 0; i < docs.length; i++) {
+        lines = docs[i].trim().split('\n');
+        res += lines[lines.length - 1] + '\n';
+    }
+    res = res.slice(0, res.length - 1);
+    return res;
+}
+
+
+export function getDetailedSkills() {
+    let res = 'namespace skills {';
+    let docs = getDocstrings();
+    for (let i = 0; i < docs.length; i++) {
+        res += '\t' + docs[i] + '\n\n';
+    }
+    res += '}';
+    return res;
+}

utils/gpt.js

@@ -0,0 +1,42 @@
+import { Configuration, OpenAIApi } from "openai";
+
+
+var openAiConfig = null;
+if (process.env.OPENAI_ORG_ID) {
+    openAiConfig = new Configuration({
+        organization: process.env.OPENAI_ORG_ID,
+        apiKey: process.env.OPENAI_API_KEY,
+    });
+} else {
+    openAiConfig = new Configuration({
+        apiKey: process.env.OPENAI_API_KEY,
+    });
+}
+const openai = new OpenAIApi(openAiConfig);
+
+
+export async function sendRequest(turns, systemMessage) {
+
+    let messages = [{"role": "system", "content": systemMessage}];
+    for (let i = 0; i < turns.length; i++) {
+        if (i % 2 == 0) {
+            messages.push({"role": "user", "content": turns[i]});
+        } else {
+            messages.push({"role": "assistant", "content": turns[i]});
+        }
+    }
+
+    let res = null;
+    try {
+        let completion = await openai.createChatCompletion({
+            model: "gpt-3.5-turbo",
+            messages: messages,
+        });
+        res = completion.data.choices[0].message.content;
+    }
+    catch (err) {
+        console.log(err);
+        res = "I'm sorry, I don't know how to respond to that.";
+    }
+    return res;
+}

utils/history.js

@@ -0,0 +1,14 @@
+
+
+
+var messages = [];
+
+
+export function addEvent(source, message) {
+    messages.push({source, message});
+}
+
+
+export function getHistory() {
+    return messages;
+}

utils/mcdata.js

@@ -0,0 +1,8 @@
+import minecraftData from 'minecraft-data';
+var mcdata = minecraftData("1.19.3");
+
+
+
+export function getItemId(item) {
+    return mcdata.itemsByName[item_type].id;
+}

utils/skills.js

@@ -0,0 +1,45 @@
+import { getItemId } from "./mcdata.js";
+import pf from 'mineflayer-pathfinder';
+
+
+export function getDocstrings() {
+    return [
+`/**
+  * Attempt to craft the given item.
+  * @param {MinecraftBot} bot, reference to the minecraft bot.
+  * @param {string} item_name, the item name to craft.
+  * @returns {Promise<boolean>} true if the item was crafted, false otherwise.
+  * @example
+  * await skills.CraftItem(bot, "wooden_pickaxe");
+  **/
+async function CraftItem(bot: MinecraftBot, item_name: string): Promise<boolean>`,
+`/**
+  * Navigate to the given player.
+  * @param {MinecraftBot} bot, reference to the minecraft bot.
+  * @param {string} username, the username of the player to navigate to.
+  * @returns {Promise<boolean>} true if the player was found, false otherwise.
+  * @example
+  * await skills.GoToPlayer(bot, "player");
+  **/
+async function GoToPlayer(bot: MinecraftBot, username: string): Promise<boolean>`
+    ]
+}
+
+
+export async function CraftItem(bot, itemName) {
+    let recipes = bot.recipesFor(getItemId(itemName), null, 1, null);  // TODO add crafting table as final arg
+    await bot.craft(recipes[0], 1, null);
+    return true;
+}
+
+
+export async function GoToPlayer(bot, username) {
+    let player = bot.players[username].entity
+    if (!player)
+        return false;
+
+    bot.pathfinder.setMovements(new pf.Movements(bot));
+    let pos = player.position;
+    bot.pathfinder.setGoal(new pf.goals.GoalNear(pos.x, pos.y, pos.z, 1));
+    return true;
+}