diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1dcef2d --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +node_modules +.env \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..eb75ff7 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,34 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Electron: Main", + "type": "node", + "request": "launch", + "program": "${workspaceFolder}/node_modules/electron/dist/electron.js", + "args": ["${workspaceFolder}/dist/main/main.js"], + "outFiles": ["${workspaceFolder}/dist/**/*.js"], + "cwd": "${workspaceFolder}", + "sourceMaps": true, + "protocol": "inspector", + "windows": { + "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron.cmd" + }, + "linux": { + "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron" + }, + "mac": { + "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron" + } + }, + { + "name": "Electron: Renderer", + "type": "chrome", + "request": "launch", + "url": "http://localhost:3000", + "webRoot": "${workspaceFolder}/src", + "sourceMaps": true + } + ] + } + \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..9b39e22 --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +# BotDesktop + +An AI-powered desktop automation tool that records and plays back user interactions. + +## Setup + +1. Install dependencies: +```bash +npm install +``` + +2. Create a .env file with your Azure OpenAI credentials + +3. Development: +```bash +npm run dev +``` + +4. Build: +```bash +npm run build +``` + +## Testing +```bash +npm test +``` diff --git a/dist/components/App.js b/dist/components/App.js new file mode 100644 index 0000000..5128888 --- /dev/null +++ b/dist/components/App.js @@ -0,0 +1,67 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const react_1 = __importStar(require("react")); +const recorder_service_1 = require("../services/recorder.service"); +const player_service_1 = require("../services/player.service"); +const recorder = new recorder_service_1.RecorderService(); +const player = new player_service_1.PlayerService(); +const App = () => { + const [recording, setRecording] = (0, react_1.useState)(false); + const [basicCode, setBasicCode] = (0, react_1.useState)(''); + const handleStartRecording = async () => { + setRecording(true); + await recorder.startRecording(); + }; + const handleStopRecording = async () => { + setRecording(false); + const code = await recorder.stopRecording(); + setBasicCode(code); + // Save to file + const blob = new Blob([code], { type: 'text/plain' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'automation.bas'; + a.click(); + }; + const handlePlayback = async () => { + try { + await player.executeBasicCode(basicCode); + } + catch (error) { + console.error('Playback error:', error); + } + }; + return (react_1.default.createElement("div", { className: "p-4" }, + react_1.default.createElement("h1", { className: "text-2xl font-bold mb-4" }, "BotDesktop Automation"), + react_1.default.createElement("div", { className: "space-x-4 mb-4" }, + react_1.default.createElement("button", { className: `px-4 py-2 rounded ${recording ? 'bg-red-500' : 'bg-blue-500'} text-white`, onClick: recording ? handleStopRecording : handleStartRecording }, recording ? 'Stop Recording' : 'Start Recording'), + react_1.default.createElement("button", { className: "px-4 py-2 rounded bg-green-500 text-white", onClick: handlePlayback, disabled: !basicCode }, "Play Recording")), + react_1.default.createElement("div", { className: "mt-4" }, + react_1.default.createElement("h2", { className: "text-xl font-bold mb-2" }, "Generated BASIC Code:"), + react_1.default.createElement("pre", { className: "bg-gray-100 p-2 rounded border" }, basicCode)))); +}; +exports.default = App; diff --git a/dist/main/main.js b/dist/main/main.js new file mode 100644 index 0000000..8cad8db --- /dev/null +++ b/dist/main/main.js @@ -0,0 +1,72 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const electron_1 = require("electron"); +const path = __importStar(require("path")); +const recorder_service_1 = require("../services/recorder.service"); +const player_service_1 = require("../services/player.service"); +const recorder = new recorder_service_1.RecorderService(); +const player = new player_service_1.PlayerService(); +function createWindow() { + const mainWindow = new electron_1.BrowserWindow({ + width: 1200, + height: 800, + webPreferences: { + nodeIntegration: true, + contextIsolation: false, + preload: path.join(__dirname, '../preload/preload.js') + } + }); + if (process.env.NODE_ENV === 'development') { + mainWindow.loadURL('http://localhost:8080'); + mainWindow.webContents.openDevTools(); + } + else { + mainWindow.loadFile(path.join(__dirname, '../../src/renderer/index.html')); + } +} +electron_1.app.whenReady().then(createWindow); +electron_1.app.on('window-all-closed', () => { + if (process.platform !== 'darwin') { + electron_1.app.quit(); + } +}); +electron_1.app.on('activate', () => { + if (electron_1.BrowserWindow.getAllWindows().length === 0) { + createWindow(); + } +}); +electron_1.ipcMain.handle('mouse-event', recorder.mouseHandleEvent.bind(recorder)); +electron_1.ipcMain.handle('keyboard-event', recorder.keyboardHandleEvent.bind(recorder)); +electron_1.ipcMain.handle('screenshot-captured', recorder.screenshotHandleEvent.bind(recorder)); +electron_1.ipcMain.handle('start-recording', async () => { + await recorder.startRecording(); +}); +electron_1.ipcMain.handle('stop-recording', async () => { + return await recorder.stopRecording(); +}); +electron_1.ipcMain.handle('execute-basic-code', async (_, code) => { + await player.executeBasicCode(code); +}); diff --git a/dist/preload/preload.js b/dist/preload/preload.js new file mode 100644 index 0000000..57f3e78 --- /dev/null +++ b/dist/preload/preload.js @@ -0,0 +1 @@ +// Preload script goes here diff --git a/dist/renderer/index.js b/dist/renderer/index.js new file mode 100644 index 0000000..7872e63 --- /dev/null +++ b/dist/renderer/index.js @@ -0,0 +1,10 @@ +"use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const react_1 = __importDefault(require("react")); +const client_1 = __importDefault(require("react-dom/client")); +const App_1 = __importDefault(require("../components/App")); +client_1.default.createRoot(document.getElementById('root')).render(react_1.default.createElement(react_1.default.StrictMode, null, + react_1.default.createElement(App_1.default, null))); diff --git a/dist/services/openai.service.js b/dist/services/openai.service.js new file mode 100644 index 0000000..bf5f539 --- /dev/null +++ b/dist/services/openai.service.js @@ -0,0 +1,56 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.OpenAIService = void 0; +const openai_1 = require("openai"); +const fs = __importStar(require("fs")); +class OpenAIService { + constructor() { + this.client = new openai_1.AzureOpenAI({ dangerouslyAllowBrowser: true, + endpoint: process.env.AZURE_OPEN_AI_ENDPOINT || '', + deployment: process.env.AZURE_OPEN_AI_IMAGE_MODEL || '', + apiVersion: process.env.OPENAI_API_VERSION || '', + apiKey: process.env.AZURE_OPEN_AI_KEY || '' + }); + } + async analyzeScreen(imagePath) { + const imageBuffer = fs.readFileSync(imagePath); + const base64Image = imageBuffer.toString('base64'); + const response = await this.client.chat.completions.create({ + model: process.env.AZURE_OPEN_AI_LLM_MODEL || '', + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Analyze this screenshot and identify all interactive elements (buttons, text fields, etc). Return their locations and identifiers.' }, + { type: 'image_url', image_url: { url: `data:image/png;base64,${base64Image}` } } + ], + }, + ], + }); + return JSON.parse(response.choices[0].message.content || '{}'); + } +} +exports.OpenAIService = OpenAIService; diff --git a/dist/services/player.service.js b/dist/services/player.service.js new file mode 100644 index 0000000..1b7512b --- /dev/null +++ b/dist/services/player.service.js @@ -0,0 +1,68 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PlayerService = void 0; +const electron_1 = require("electron"); +const openai_service_1 = require("./openai.service"); +class PlayerService { + constructor() { + this.openAIService = new openai_service_1.OpenAIService(); + } + async executeBasicCode(code) { + const lines = code.split('\n'); + for (const line of lines) { + if (line.trim().startsWith('REM') || line.trim() === '') + continue; + const match = line.match(/^\d+\s+(\w+)\s+"([^"]+)"(?:\s+"([^"]+)")?/); + if (!match) + continue; + const [_, command, identifier, value] = match; + await this.executeCommand(command, identifier, value); + await new Promise(resolve => setTimeout(resolve, 500)); + } + } + async executeCommand(command, identifier, value) { + // Capture current screen + const screenshotPath = await this.captureScreen(); + const analysis = await this.openAIService.analyzeScreen(screenshotPath); + const element = analysis.elements.find(e => e.identifier === identifier); + if (!element) + throw new Error(`Element not found: ${identifier}`); + // Calculate center point of element + const centerX = element.bounds.x + element.bounds.width / 2; + const centerY = element.bounds.y + element.bounds.height / 2; + switch (command) { + case 'CLICK': + await this.simulateClick(centerX, centerY); + break; + case 'TYPE': + await this.simulateClick(centerX, centerY); + await this.simulateTyping(value || ''); + break; + } + } + async captureScreen() { + return new Promise((resolve, reject) => { + electron_1.ipcMain.once('screen-captured', (_, screenshotPath) => { + resolve(screenshotPath); + }); + electron_1.ipcMain.emit('capture-screen'); + }); + } + async simulateClick(x, y) { + return new Promise((resolve) => { + electron_1.ipcMain.once('click-completed', () => { + resolve(); + }); + electron_1.ipcMain.emit('simulate-click', { x, y }); + }); + } + async simulateTyping(text) { + return new Promise((resolve) => { + electron_1.ipcMain.once('typing-completed', () => { + resolve(); + }); + electron_1.ipcMain.emit('simulate-typing', { text }); + }); + } +} +exports.PlayerService = PlayerService; diff --git a/dist/services/recorder.service.js b/dist/services/recorder.service.js new file mode 100644 index 0000000..54e1011 --- /dev/null +++ b/dist/services/recorder.service.js @@ -0,0 +1,93 @@ +"use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.RecorderService = void 0; +const electron_1 = require("electron"); +const dotenv_1 = __importDefault(require("dotenv")); +dotenv_1.default.config(); +const openai_service_1 = require("./openai.service"); +class RecorderService { + constructor() { + this.events = []; + this.recording = false; + this.currentScreenshot = ''; + this.openAIService = new openai_service_1.OpenAIService(); + } + async startRecording() { + this.recording = true; + this.events = []; + this.requestScreenshot(); + } + stopRecording() { + this.recording = false; + return this.generateBasicCode(); + } + requestScreenshot() { + // Notify renderer process to capture a screenshot + const allWebContents = electron_1.screen.getAllDisplays(); + allWebContents.forEach((webContents) => { + //@ts-ignores + webContents.send('request-screenshot'); + }); + } + async screenshotHandleEvent(_, screenshot) { + this.currentScreenshot = screenshot; // Store the screenshot as a base64 image + } + async mouseHandleEvent(_, event) { + if (!this.recording) + return; + const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot); + const element = this.findElementAtPosition(analysis, event.x, event.y); + if (element) { + this.events.push({ + type: 'click', + identifier: element.identifier, + timestamp: Date.now(), + }); + } + } + async keyboardHandleEvent(_, event) { + if (!this.recording) + return; + const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot); + const focusedElement = this.findFocusedElement(analysis); + if (focusedElement) { + this.events.push({ + type: 'type', + identifier: focusedElement.identifier, + value: event.key, + timestamp: Date.now(), + }); + } + } + findElementAtPosition(analysis, x, y) { + return analysis.elements.find((element) => { + const bounds = element.bounds; + return x >= bounds.x && x <= bounds.x + bounds.width && y >= bounds.y && y <= bounds.y + bounds.height; + }); + } + findFocusedElement(analysis) { + //@ts-ignore + return analysis.elements.find((element) => element.focused); + } + generateBasicCode() { + let basicCode = '10 REM BotDesktop Automation Script\n'; + let lineNumber = 20; + for (const event of this.events) { + switch (event.type) { + case 'click': + basicCode += `${lineNumber} CLICK "${event.identifier}"\n`; + break; + case 'type': + basicCode += `${lineNumber} TYPE "${event.identifier}" "${event.value}"\n`; + break; + } + lineNumber += 10; + } + basicCode += `${lineNumber} END\n`; + return basicCode; + } +} +exports.RecorderService = RecorderService; diff --git a/dist/services/types.js b/dist/services/types.js new file mode 100644 index 0000000..c8ad2e5 --- /dev/null +++ b/dist/services/types.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/dist/tests/services.test.js b/dist/tests/services.test.js new file mode 100644 index 0000000..59bb45a --- /dev/null +++ b/dist/tests/services.test.js @@ -0,0 +1 @@ +// Tests for services diff --git a/electron-builder.json b/electron-builder.json new file mode 100644 index 0000000..8f6078a --- /dev/null +++ b/electron-builder.json @@ -0,0 +1,20 @@ +{ + "appId": "com.botdesktop.app", + "directories": { + "output": "release/" + }, + "files": [ + "dist/**/*", + "node_modules/**/*", + "package.json" + ], + "mac": { + "target": ["dmg"] + }, + "win": { + "target": ["nsis"] + }, + "linux": { + "target": ["AppImage"] + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..0eca355 --- /dev/null +++ b/package.json @@ -0,0 +1,35 @@ +{ + "name": "bot-desktop", + "version": "1.0.0", + "description": "AI-powered desktop automation tool", + "main": "dist/main/main.js", + "scripts": { + "start": "electron .", + "dev": "concurrently \"webpack serve --mode development\" \"tsc -w -p tsconfig.electron.json\" \"electron .\"", + "build": "webpack --mode production && tsc -p tsconfig.electron.json && electron-builder", + "test": "vitest" + }, + "dependencies": { + "dotenv": "^16.4.5", + "node-global-key-listener": "^0.3.0", + "node-mouse": "^0.0.2", + "@types/node": "^20.0.0", + "@types/react": "^18.0.0", + "@types/react-dom": "^18.0.0", + "electron": "^28.0.0", + "openai": "^4.28.0", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "typescript": "^5.0.0" + }, + "devDependencies": { + "concurrently": "^8.2.2", + "electron-builder": "^24.9.1", + "html-webpack-plugin": "^5.6.0", + "ts-loader": "^9.5.1", + "vitest": "^1.2.1", + "webpack": "^5.89.0", + "webpack-cli": "^5.1.4", + "webpack-dev-server": "^4.15.1" + } +} diff --git a/src/components/App.tsx b/src/components/App.tsx new file mode 100644 index 0000000..bdf15ed --- /dev/null +++ b/src/components/App.tsx @@ -0,0 +1,67 @@ +import React, { useState } from 'react'; +import { RecorderService } from '../services/recorder.service'; +import { PlayerService } from '../services/player.service'; + +const recorder = new RecorderService(); +const player = new PlayerService(); + +const App: React.FC = () => { + const [recording, setRecording] = useState(false); + const [basicCode, setBasicCode] = useState(''); + + const handleStartRecording = async () => { + setRecording(true); + await recorder.startRecording(); + }; + + const handleStopRecording = async () => { + setRecording(false); + const code = await recorder.stopRecording(); + setBasicCode(code); + // Save to file + const blob = new Blob([code], { type: 'text/plain' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'automation.bas'; + a.click(); + }; + + const handlePlayback = async () => { + try { + await player.executeBasicCode(basicCode); + } catch (error) { + console.error('Playback error:', error); + } + }; + + return ( +
{basicCode}
+