From 146ad03aea616c1b2ddaf0bd6be16d26c9c71194 Mon Sep 17 00:00:00 2001 From: "me@rodrigorodriguez.com" Date: Sat, 26 Oct 2024 13:05:56 -0300 Subject: [PATCH] new(all): Initial import. --- .gitignore | 2 + .vscode/launch.json | 34 ++++++++++ README.md | 27 ++++++++ dist/components/App.js | 67 +++++++++++++++++++ dist/main/main.js | 72 +++++++++++++++++++++ dist/preload/preload.js | 1 + dist/renderer/index.js | 10 +++ dist/services/openai.service.js | 56 ++++++++++++++++ dist/services/player.service.js | 68 ++++++++++++++++++++ dist/services/recorder.service.js | 93 +++++++++++++++++++++++++++ dist/services/types.js | 2 + dist/tests/services.test.js | 1 + electron-builder.json | 20 ++++++ package.json | 35 ++++++++++ src/components/App.tsx | 67 +++++++++++++++++++ src/main/main.ts | 58 +++++++++++++++++ src/preload/preload.ts | 1 + src/renderer/index.html | 11 ++++ src/renderer/index.tsx | 14 ++++ src/services/openai.service.ts | 36 +++++++++++ src/services/player.service.ts | 80 +++++++++++++++++++++++ src/services/recorder.service.ts | 103 ++++++++++++++++++++++++++++++ src/services/types.ts | 15 +++++ src/tests/services.test.ts | 1 + tsconfig.electron.json | 7 ++ tsconfig.json | 22 +++++++ webpack.config.js | 28 ++++++++ 27 files changed, 931 insertions(+) create mode 100644 .gitignore create mode 100644 .vscode/launch.json create mode 100644 README.md create mode 100644 dist/components/App.js create mode 100644 dist/main/main.js create mode 100644 dist/preload/preload.js create mode 100644 dist/renderer/index.js create mode 100644 dist/services/openai.service.js create mode 100644 dist/services/player.service.js create mode 100644 dist/services/recorder.service.js create mode 100644 dist/services/types.js create mode 100644 dist/tests/services.test.js create mode 100644 electron-builder.json create mode 100644 package.json create mode 100644 src/components/App.tsx create mode 100644 src/main/main.ts create mode 100644 src/preload/preload.ts create mode 100644 src/renderer/index.html create mode 100644 src/renderer/index.tsx create mode 100644 src/services/openai.service.ts create mode 100644 src/services/player.service.ts create mode 100644 src/services/recorder.service.ts create mode 100644 src/services/types.ts create mode 100644 src/tests/services.test.ts create mode 100644 tsconfig.electron.json create mode 100644 tsconfig.json create mode 100644 webpack.config.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1dcef2d --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +node_modules +.env \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..eb75ff7 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,34 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Electron: Main", + "type": "node", + "request": "launch", + "program": "${workspaceFolder}/node_modules/electron/dist/electron.js", + "args": ["${workspaceFolder}/dist/main/main.js"], + "outFiles": ["${workspaceFolder}/dist/**/*.js"], + "cwd": "${workspaceFolder}", + "sourceMaps": true, + "protocol": "inspector", + "windows": { + "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron.cmd" + }, + "linux": { + "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron" + }, + "mac": { + "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron" + } + }, + { + "name": "Electron: Renderer", + "type": "chrome", + "request": "launch", + "url": "http://localhost:3000", + "webRoot": "${workspaceFolder}/src", + "sourceMaps": true + } + ] + } + \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..9b39e22 --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +# BotDesktop + +An AI-powered desktop automation tool that records and plays back user interactions. + +## Setup + +1. Install dependencies: +```bash +npm install +``` + +2. Create a .env file with your Azure OpenAI credentials + +3. Development: +```bash +npm run dev +``` + +4. Build: +```bash +npm run build +``` + +## Testing +```bash +npm test +``` diff --git a/dist/components/App.js b/dist/components/App.js new file mode 100644 index 0000000..5128888 --- /dev/null +++ b/dist/components/App.js @@ -0,0 +1,67 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const react_1 = __importStar(require("react")); +const recorder_service_1 = require("../services/recorder.service"); +const player_service_1 = require("../services/player.service"); +const recorder = new recorder_service_1.RecorderService(); +const player = new player_service_1.PlayerService(); +const App = () => { + const [recording, setRecording] = (0, react_1.useState)(false); + const [basicCode, setBasicCode] = (0, react_1.useState)(''); + const handleStartRecording = async () => { + setRecording(true); + await recorder.startRecording(); + }; + const handleStopRecording = async () => { + setRecording(false); + const code = await recorder.stopRecording(); + setBasicCode(code); + // Save to file + const blob = new Blob([code], { type: 'text/plain' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'automation.bas'; + a.click(); + }; + const handlePlayback = async () => { + try { + await player.executeBasicCode(basicCode); + } + catch (error) { + console.error('Playback error:', error); + } + }; + return (react_1.default.createElement("div", { className: "p-4" }, + react_1.default.createElement("h1", { className: "text-2xl font-bold mb-4" }, "BotDesktop Automation"), + react_1.default.createElement("div", { className: "space-x-4 mb-4" }, + react_1.default.createElement("button", { className: `px-4 py-2 rounded ${recording ? 'bg-red-500' : 'bg-blue-500'} text-white`, onClick: recording ? handleStopRecording : handleStartRecording }, recording ? 'Stop Recording' : 'Start Recording'), + react_1.default.createElement("button", { className: "px-4 py-2 rounded bg-green-500 text-white", onClick: handlePlayback, disabled: !basicCode }, "Play Recording")), + react_1.default.createElement("div", { className: "mt-4" }, + react_1.default.createElement("h2", { className: "text-xl font-bold mb-2" }, "Generated BASIC Code:"), + react_1.default.createElement("pre", { className: "bg-gray-100 p-2 rounded border" }, basicCode)))); +}; +exports.default = App; diff --git a/dist/main/main.js b/dist/main/main.js new file mode 100644 index 0000000..8cad8db --- /dev/null +++ b/dist/main/main.js @@ -0,0 +1,72 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const electron_1 = require("electron"); +const path = __importStar(require("path")); +const recorder_service_1 = require("../services/recorder.service"); +const player_service_1 = require("../services/player.service"); +const recorder = new recorder_service_1.RecorderService(); +const player = new player_service_1.PlayerService(); +function createWindow() { + const mainWindow = new electron_1.BrowserWindow({ + width: 1200, + height: 800, + webPreferences: { + nodeIntegration: true, + contextIsolation: false, + preload: path.join(__dirname, '../preload/preload.js') + } + }); + if (process.env.NODE_ENV === 'development') { + mainWindow.loadURL('http://localhost:8080'); + mainWindow.webContents.openDevTools(); + } + else { + mainWindow.loadFile(path.join(__dirname, '../../src/renderer/index.html')); + } +} +electron_1.app.whenReady().then(createWindow); +electron_1.app.on('window-all-closed', () => { + if (process.platform !== 'darwin') { + electron_1.app.quit(); + } +}); +electron_1.app.on('activate', () => { + if (electron_1.BrowserWindow.getAllWindows().length === 0) { + createWindow(); + } +}); +electron_1.ipcMain.handle('mouse-event', recorder.mouseHandleEvent.bind(recorder)); +electron_1.ipcMain.handle('keyboard-event', recorder.keyboardHandleEvent.bind(recorder)); +electron_1.ipcMain.handle('screenshot-captured', recorder.screenshotHandleEvent.bind(recorder)); +electron_1.ipcMain.handle('start-recording', async () => { + await recorder.startRecording(); +}); +electron_1.ipcMain.handle('stop-recording', async () => { + return await recorder.stopRecording(); +}); +electron_1.ipcMain.handle('execute-basic-code', async (_, code) => { + await player.executeBasicCode(code); +}); diff --git a/dist/preload/preload.js b/dist/preload/preload.js new file mode 100644 index 0000000..57f3e78 --- /dev/null +++ b/dist/preload/preload.js @@ -0,0 +1 @@ +// Preload script goes here diff --git a/dist/renderer/index.js b/dist/renderer/index.js new file mode 100644 index 0000000..7872e63 --- /dev/null +++ b/dist/renderer/index.js @@ -0,0 +1,10 @@ +"use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const react_1 = __importDefault(require("react")); +const client_1 = __importDefault(require("react-dom/client")); +const App_1 = __importDefault(require("../components/App")); +client_1.default.createRoot(document.getElementById('root')).render(react_1.default.createElement(react_1.default.StrictMode, null, + react_1.default.createElement(App_1.default, null))); diff --git a/dist/services/openai.service.js b/dist/services/openai.service.js new file mode 100644 index 0000000..bf5f539 --- /dev/null +++ b/dist/services/openai.service.js @@ -0,0 +1,56 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.OpenAIService = void 0; +const openai_1 = require("openai"); +const fs = __importStar(require("fs")); +class OpenAIService { + constructor() { + this.client = new openai_1.AzureOpenAI({ dangerouslyAllowBrowser: true, + endpoint: process.env.AZURE_OPEN_AI_ENDPOINT || '', + deployment: process.env.AZURE_OPEN_AI_IMAGE_MODEL || '', + apiVersion: process.env.OPENAI_API_VERSION || '', + apiKey: process.env.AZURE_OPEN_AI_KEY || '' + }); + } + async analyzeScreen(imagePath) { + const imageBuffer = fs.readFileSync(imagePath); + const base64Image = imageBuffer.toString('base64'); + const response = await this.client.chat.completions.create({ + model: process.env.AZURE_OPEN_AI_LLM_MODEL || '', + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Analyze this screenshot and identify all interactive elements (buttons, text fields, etc). Return their locations and identifiers.' }, + { type: 'image_url', image_url: { url: `data:image/png;base64,${base64Image}` } } + ], + }, + ], + }); + return JSON.parse(response.choices[0].message.content || '{}'); + } +} +exports.OpenAIService = OpenAIService; diff --git a/dist/services/player.service.js b/dist/services/player.service.js new file mode 100644 index 0000000..1b7512b --- /dev/null +++ b/dist/services/player.service.js @@ -0,0 +1,68 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PlayerService = void 0; +const electron_1 = require("electron"); +const openai_service_1 = require("./openai.service"); +class PlayerService { + constructor() { + this.openAIService = new openai_service_1.OpenAIService(); + } + async executeBasicCode(code) { + const lines = code.split('\n'); + for (const line of lines) { + if (line.trim().startsWith('REM') || line.trim() === '') + continue; + const match = line.match(/^\d+\s+(\w+)\s+"([^"]+)"(?:\s+"([^"]+)")?/); + if (!match) + continue; + const [_, command, identifier, value] = match; + await this.executeCommand(command, identifier, value); + await new Promise(resolve => setTimeout(resolve, 500)); + } + } + async executeCommand(command, identifier, value) { + // Capture current screen + const screenshotPath = await this.captureScreen(); + const analysis = await this.openAIService.analyzeScreen(screenshotPath); + const element = analysis.elements.find(e => e.identifier === identifier); + if (!element) + throw new Error(`Element not found: ${identifier}`); + // Calculate center point of element + const centerX = element.bounds.x + element.bounds.width / 2; + const centerY = element.bounds.y + element.bounds.height / 2; + switch (command) { + case 'CLICK': + await this.simulateClick(centerX, centerY); + break; + case 'TYPE': + await this.simulateClick(centerX, centerY); + await this.simulateTyping(value || ''); + break; + } + } + async captureScreen() { + return new Promise((resolve, reject) => { + electron_1.ipcMain.once('screen-captured', (_, screenshotPath) => { + resolve(screenshotPath); + }); + electron_1.ipcMain.emit('capture-screen'); + }); + } + async simulateClick(x, y) { + return new Promise((resolve) => { + electron_1.ipcMain.once('click-completed', () => { + resolve(); + }); + electron_1.ipcMain.emit('simulate-click', { x, y }); + }); + } + async simulateTyping(text) { + return new Promise((resolve) => { + electron_1.ipcMain.once('typing-completed', () => { + resolve(); + }); + electron_1.ipcMain.emit('simulate-typing', { text }); + }); + } +} +exports.PlayerService = PlayerService; diff --git a/dist/services/recorder.service.js b/dist/services/recorder.service.js new file mode 100644 index 0000000..54e1011 --- /dev/null +++ b/dist/services/recorder.service.js @@ -0,0 +1,93 @@ +"use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.RecorderService = void 0; +const electron_1 = require("electron"); +const dotenv_1 = __importDefault(require("dotenv")); +dotenv_1.default.config(); +const openai_service_1 = require("./openai.service"); +class RecorderService { + constructor() { + this.events = []; + this.recording = false; + this.currentScreenshot = ''; + this.openAIService = new openai_service_1.OpenAIService(); + } + async startRecording() { + this.recording = true; + this.events = []; + this.requestScreenshot(); + } + stopRecording() { + this.recording = false; + return this.generateBasicCode(); + } + requestScreenshot() { + // Notify renderer process to capture a screenshot + const allWebContents = electron_1.screen.getAllDisplays(); + allWebContents.forEach((webContents) => { + //@ts-ignores + webContents.send('request-screenshot'); + }); + } + async screenshotHandleEvent(_, screenshot) { + this.currentScreenshot = screenshot; // Store the screenshot as a base64 image + } + async mouseHandleEvent(_, event) { + if (!this.recording) + return; + const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot); + const element = this.findElementAtPosition(analysis, event.x, event.y); + if (element) { + this.events.push({ + type: 'click', + identifier: element.identifier, + timestamp: Date.now(), + }); + } + } + async keyboardHandleEvent(_, event) { + if (!this.recording) + return; + const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot); + const focusedElement = this.findFocusedElement(analysis); + if (focusedElement) { + this.events.push({ + type: 'type', + identifier: focusedElement.identifier, + value: event.key, + timestamp: Date.now(), + }); + } + } + findElementAtPosition(analysis, x, y) { + return analysis.elements.find((element) => { + const bounds = element.bounds; + return x >= bounds.x && x <= bounds.x + bounds.width && y >= bounds.y && y <= bounds.y + bounds.height; + }); + } + findFocusedElement(analysis) { + //@ts-ignore + return analysis.elements.find((element) => element.focused); + } + generateBasicCode() { + let basicCode = '10 REM BotDesktop Automation Script\n'; + let lineNumber = 20; + for (const event of this.events) { + switch (event.type) { + case 'click': + basicCode += `${lineNumber} CLICK "${event.identifier}"\n`; + break; + case 'type': + basicCode += `${lineNumber} TYPE "${event.identifier}" "${event.value}"\n`; + break; + } + lineNumber += 10; + } + basicCode += `${lineNumber} END\n`; + return basicCode; + } +} +exports.RecorderService = RecorderService; diff --git a/dist/services/types.js b/dist/services/types.js new file mode 100644 index 0000000..c8ad2e5 --- /dev/null +++ b/dist/services/types.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/dist/tests/services.test.js b/dist/tests/services.test.js new file mode 100644 index 0000000..59bb45a --- /dev/null +++ b/dist/tests/services.test.js @@ -0,0 +1 @@ +// Tests for services diff --git a/electron-builder.json b/electron-builder.json new file mode 100644 index 0000000..8f6078a --- /dev/null +++ b/electron-builder.json @@ -0,0 +1,20 @@ +{ + "appId": "com.botdesktop.app", + "directories": { + "output": "release/" + }, + "files": [ + "dist/**/*", + "node_modules/**/*", + "package.json" + ], + "mac": { + "target": ["dmg"] + }, + "win": { + "target": ["nsis"] + }, + "linux": { + "target": ["AppImage"] + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..0eca355 --- /dev/null +++ b/package.json @@ -0,0 +1,35 @@ +{ + "name": "bot-desktop", + "version": "1.0.0", + "description": "AI-powered desktop automation tool", + "main": "dist/main/main.js", + "scripts": { + "start": "electron .", + "dev": "concurrently \"webpack serve --mode development\" \"tsc -w -p tsconfig.electron.json\" \"electron .\"", + "build": "webpack --mode production && tsc -p tsconfig.electron.json && electron-builder", + "test": "vitest" + }, + "dependencies": { + "dotenv": "^16.4.5", + "node-global-key-listener": "^0.3.0", + "node-mouse": "^0.0.2", + "@types/node": "^20.0.0", + "@types/react": "^18.0.0", + "@types/react-dom": "^18.0.0", + "electron": "^28.0.0", + "openai": "^4.28.0", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "typescript": "^5.0.0" + }, + "devDependencies": { + "concurrently": "^8.2.2", + "electron-builder": "^24.9.1", + "html-webpack-plugin": "^5.6.0", + "ts-loader": "^9.5.1", + "vitest": "^1.2.1", + "webpack": "^5.89.0", + "webpack-cli": "^5.1.4", + "webpack-dev-server": "^4.15.1" + } +} diff --git a/src/components/App.tsx b/src/components/App.tsx new file mode 100644 index 0000000..bdf15ed --- /dev/null +++ b/src/components/App.tsx @@ -0,0 +1,67 @@ +import React, { useState } from 'react'; +import { RecorderService } from '../services/recorder.service'; +import { PlayerService } from '../services/player.service'; + +const recorder = new RecorderService(); +const player = new PlayerService(); + +const App: React.FC = () => { + const [recording, setRecording] = useState(false); + const [basicCode, setBasicCode] = useState(''); + + const handleStartRecording = async () => { + setRecording(true); + await recorder.startRecording(); + }; + + const handleStopRecording = async () => { + setRecording(false); + const code = await recorder.stopRecording(); + setBasicCode(code); + // Save to file + const blob = new Blob([code], { type: 'text/plain' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'automation.bas'; + a.click(); + }; + + const handlePlayback = async () => { + try { + await player.executeBasicCode(basicCode); + } catch (error) { + console.error('Playback error:', error); + } + }; + + return ( +
+

BotDesktop Automation

+ +
+ + + +
+ +
+

Generated BASIC Code:

+ 
{basicCode}
+
+
+ ); +}; + +export default App; diff --git a/src/main/main.ts b/src/main/main.ts new file mode 100644 index 0000000..17f228f --- /dev/null +++ b/src/main/main.ts @@ -0,0 +1,58 @@ +import { app, BrowserWindow, ipcMain } from 'electron'; +import * as path from 'path'; + +import { RecorderService } from '../services/recorder.service'; +import { PlayerService } from '../services/player.service'; + +const recorder = new RecorderService(); +const player = new PlayerService(); + +function createWindow() { + const mainWindow = new BrowserWindow({ + width: 1200, + height: 800, + webPreferences: { + nodeIntegration: true, + contextIsolation: false, + preload: path.join(__dirname, '../preload/preload.js') + } + }); + + if (process.env.NODE_ENV === 'development') { + mainWindow.loadURL('http://localhost:8080'); + mainWindow.webContents.openDevTools(); + } else { + mainWindow.loadFile(path.join(__dirname, '../../src/renderer/index.html')); + } +} + +app.whenReady().then(createWindow); + +app.on('window-all-closed', () => { + if (process.platform !== 'darwin') { + app.quit(); + } +}); + +app.on('activate', () => { + if (BrowserWindow.getAllWindows().length === 0) { + createWindow(); + } +}); + +ipcMain.handle('mouse-event', recorder.mouseHandleEvent.bind(recorder)); +ipcMain.handle('keyboard-event', recorder.keyboardHandleEvent.bind(recorder)); +ipcMain.handle('screenshot-captured', recorder.screenshotHandleEvent.bind(recorder)); + + +ipcMain.handle('start-recording', async () => { + await recorder.startRecording(); +}); + +ipcMain.handle('stop-recording', async () => { + return await recorder.stopRecording(); +}); + +ipcMain.handle('execute-basic-code', async (_, code: string) => { + await player.executeBasicCode(code); +}); diff --git a/src/preload/preload.ts b/src/preload/preload.ts new file mode 100644 index 0000000..57f3e78 --- /dev/null +++ b/src/preload/preload.ts @@ -0,0 +1 @@ +// Preload script goes here diff --git a/src/renderer/index.html b/src/renderer/index.html new file mode 100644 index 0000000..2001fb6 --- /dev/null +++ b/src/renderer/index.html @@ -0,0 +1,11 @@ + + + + + BotDesktop + + + +
+ + diff --git a/src/renderer/index.tsx b/src/renderer/index.tsx new file mode 100644 index 0000000..b822859 --- /dev/null +++ b/src/renderer/index.tsx @@ -0,0 +1,14 @@ +import React from 'react'; +import ReactDOM from 'react-dom/client'; + + + +import App from '../components/App'; + +ReactDOM.createRoot( + document.getElementById('root') as HTMLElement +).render( + + + +); diff --git a/src/services/openai.service.ts b/src/services/openai.service.ts new file mode 100644 index 0000000..58392cb --- /dev/null +++ b/src/services/openai.service.ts @@ -0,0 +1,36 @@ +import { AzureOpenAI } from 'openai'; +import * as fs from 'fs'; +import { ScreenAnalysis } from './types'; + +export class OpenAIService { + private client: AzureOpenAI; + + constructor() { + this.client = new AzureOpenAI({ dangerouslyAllowBrowser: true, + endpoint: process.env.AZURE_OPEN_AI_ENDPOINT || '', + deployment: process.env.AZURE_OPEN_AI_IMAGE_MODEL || '', + apiVersion: process.env.OPENAI_API_VERSION || '', + apiKey: process.env.AZURE_OPEN_AI_KEY || '' + }); + } + + async analyzeScreen(imagePath: string): Promise { + const imageBuffer = fs.readFileSync(imagePath); + const base64Image = imageBuffer.toString('base64'); + + const response = await this.client.chat.completions.create({ + model: process.env.AZURE_OPEN_AI_LLM_MODEL || '', + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Analyze this screenshot and identify all interactive elements (buttons, text fields, etc). Return their locations and identifiers.' }, + { type: 'image_url', image_url: { url: `data:image/png;base64,${base64Image}` } } + ], + }, + ], + }); + + return JSON.parse(response.choices[0].message.content || '{}'); + } +} diff --git a/src/services/player.service.ts b/src/services/player.service.ts new file mode 100644 index 0000000..ad56b7e --- /dev/null +++ b/src/services/player.service.ts @@ -0,0 +1,80 @@ +import { ipcMain } from 'electron'; +import { AutomationEvent, ScreenAnalysis } from './types'; +import { OpenAIService } from './openai.service'; + +export class PlayerService { + private openAIService: OpenAIService; + + constructor() { + this.openAIService = new OpenAIService(); + } + + async executeBasicCode(code: string) { + const lines = code.split('\n'); + + for (const line of lines) { + if (line.trim().startsWith('REM') || line.trim() === '') continue; + + const match = line.match(/^\d+\s+(\w+)\s+"([^"]+)"(?:\s+"([^"]+)")?/); + if (!match) continue; + + const [_, command, identifier, value] = match; + await this.executeCommand(command, identifier, value); + await new Promise(resolve => setTimeout(resolve, 500)); + } + } + + private async executeCommand(command: string, identifier: string, value?: string) { + // Capture current screen + const screenshotPath = await this.captureScreen(); + + const analysis = await this.openAIService.analyzeScreen(screenshotPath); + const element = analysis.elements.find(e => e.identifier === identifier); + + if (!element) throw new Error(`Element not found: ${identifier}`); + + // Calculate center point of element + const centerX = element.bounds.x + element.bounds.width/2; + const centerY = element.bounds.y + element.bounds.height/2; + + switch (command) { + case 'CLICK': + await this.simulateClick(centerX, centerY); + break; + case 'TYPE': + await this.simulateClick(centerX, centerY); + await this.simulateTyping(value || ''); + break; + } + } + + private async captureScreen(): Promise { + return new Promise((resolve, reject) => { + ipcMain.once('screen-captured', (_, screenshotPath) => { + resolve(screenshotPath); + }); + + ipcMain.emit('capture-screen'); + }); + } + + private async simulateClick(x: number, y: number): Promise { + return new Promise((resolve) => { + ipcMain.once('click-completed', () => { + resolve(); + }); + + ipcMain.emit('simulate-click', { x, y }); + }); + } + + private async simulateTyping(text: string): Promise { + return new Promise((resolve) => { + ipcMain.once('typing-completed', () => { + resolve(); + }); + + ipcMain.emit('simulate-typing', { text }); + }); + } +} \ No newline at end of file diff --git a/src/services/recorder.service.ts b/src/services/recorder.service.ts new file mode 100644 index 0000000..93cedd9 --- /dev/null +++ b/src/services/recorder.service.ts @@ -0,0 +1,103 @@ +import { screen, ipcMain } from 'electron'; +import { AutomationEvent, ScreenAnalysis } from './types'; +import dotenv from 'dotenv'; +dotenv.config(); +import { OpenAIService } from './openai.service'; + +export class RecorderService { + private events: AutomationEvent[] = []; + private recording: boolean = false; + private openAIService: OpenAIService; + private currentScreenshot: string = ''; + + constructor() { + this.openAIService = new OpenAIService(); + } + + public async startRecording() { + this.recording = true; + this.events = []; + this.requestScreenshot(); + } + + public stopRecording(): string { + this.recording = false; + return this.generateBasicCode(); + } + + private requestScreenshot() { + // Notify renderer process to capture a screenshot + const allWebContents = screen.getAllDisplays(); + allWebContents.forEach((webContents) => { + //@ts-ignores + webContents.send('request-screenshot'); + }); + } + + public async screenshotHandleEvent (_: any, screenshot: string) { + this.currentScreenshot = screenshot; // Store the screenshot as a base64 image + } + + public async mouseHandleEvent(_: any, event: any) { + if (!this.recording) return; + + const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot); + const element = this.findElementAtPosition(analysis, event.x, event.y); + + if (element) { + this.events.push({ + type: 'click', + identifier: element.identifier, + timestamp: Date.now(), + }); + } + } + + public async keyboardHandleEvent(_: any, event: any) { + if (!this.recording) return; + + const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot); + const focusedElement = this.findFocusedElement(analysis); + + if (focusedElement) { + this.events.push({ + type: 'type', + identifier: focusedElement.identifier, + value: event.key, + timestamp: Date.now(), + }); + } + } + + private findElementAtPosition(analysis: ScreenAnalysis, x: number, y: number) { + return analysis.elements.find((element) => { + const bounds = element.bounds; + return x >= bounds.x && x <= bounds.x + bounds.width && y >= bounds.y && y <= bounds.y + bounds.height; + }); + } + + private findFocusedElement(analysis: ScreenAnalysis) { + //@ts-ignore + return analysis.elements.find((element) => element.focused); + } + + private generateBasicCode(): string { + let basicCode = '10 REM BotDesktop Automation Script\n'; + let lineNumber = 20; + + for (const event of this.events) { + switch (event.type) { + case 'click': + basicCode += `${lineNumber} CLICK "${event.identifier}"\n`; + break; + case 'type': + basicCode += `${lineNumber} TYPE "${event.identifier}" "${event.value}"\n`; + break; + } + lineNumber += 10; + } + + basicCode += `${lineNumber} END\n`; + return basicCode; + } +} diff --git a/src/services/types.ts b/src/services/types.ts new file mode 100644 index 0000000..5d7de19 --- /dev/null +++ b/src/services/types.ts @@ -0,0 +1,15 @@ +export interface AutomationEvent { + type: 'click' | 'type' | 'move'; + identifier: string; + value?: string; + timestamp: number; +} + +export interface ScreenAnalysis { + elements: { + identifier: string; + type: string; + bounds: { x: number, y: number, width: number, height: number }; + value?: string; + }[]; +} diff --git a/src/tests/services.test.ts b/src/tests/services.test.ts new file mode 100644 index 0000000..59bb45a --- /dev/null +++ b/src/tests/services.test.ts @@ -0,0 +1 @@ +// Tests for services diff --git a/tsconfig.electron.json b/tsconfig.electron.json new file mode 100644 index 0000000..eac7cde --- /dev/null +++ b/tsconfig.electron.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "CommonJS", + "outDir": "dist" + } +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..dc3a8d5 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "CommonJS", + "lib": ["DOM", "ES2020"], + "jsx": "react", + "strict": false, + "noImplicitAny": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist", + "rootDir": "src", + "moduleResolution": "node", + "resolveJsonModule": true, + "baseUrl": ".", + "paths": { + "@/*": ["src/*"] + } + }, + "include": ["src/**/*"], + "exclude": ["node_modules"] +} diff --git a/webpack.config.js b/webpack.config.js new file mode 100644 index 0000000..17fa3c0 --- /dev/null +++ b/webpack.config.js @@ -0,0 +1,28 @@ +const path = require('path'); +const HtmlWebpackPlugin = require('html-webpack-plugin'); + +module.exports = { + entry: './src/renderer/index.tsx', + target: 'electron-renderer', + module: { + rules: [ + { + test: /\.tsx?$/, + use: 'ts-loader', + exclude: /node_modules/, + }, + ], + }, + resolve: { + extensions: ['.tsx', '.ts', '.js'], + }, + output: { + filename: 'renderer.js', + path: path.resolve(__dirname, 'dist/renderer'), + }, + plugins: [ + new HtmlWebpackPlugin({ + template: './src/renderer/index.html' + }), + ], +}; \ No newline at end of file