new(all): Initial import.

.gitignore

@@ -0,0 +1,2 @@
+node_modules
+.env

.vscode/launch.json

@@ -0,0 +1,34 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "name": "Electron: Main",
+        "type": "node",
+        "request": "launch",
+        "program": "${workspaceFolder}/node_modules/electron/dist/electron.js",
+        "args": ["${workspaceFolder}/dist/main/main.js"],
+        "outFiles": ["${workspaceFolder}/dist/**/*.js"],
+        "cwd": "${workspaceFolder}",
+        "sourceMaps": true,
+        "protocol": "inspector",
+        "windows": {
+          "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron.cmd"
+        },
+        "linux": {
+          "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron"
+        },
+        "mac": {
+          "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/electron"
+        }
+      },
+      {
+        "name": "Electron: Renderer",
+        "type": "chrome",
+        "request": "launch",
+        "url": "http://localhost:3000",
+        "webRoot": "${workspaceFolder}/src",
+        "sourceMaps": true
+      }
+    ]
+  }
+  

README.md

@@ -0,0 +1,27 @@
+# BotDesktop
+
+An AI-powered desktop automation tool that records and plays back user interactions.
+
+## Setup
+
+1. Install dependencies:
+```bash
+npm install
+```
+
+2. Create a .env file with your Azure OpenAI credentials
+
+3. Development:
+```bash
+npm run dev
+```
+
+4. Build:
+```bash
+npm run build
+```
+
+## Testing
+```bash
+npm test
+```

dist/components/App.js

@@ -0,0 +1,67 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const react_1 = __importStar(require("react"));
+const recorder_service_1 = require("../services/recorder.service");
+const player_service_1 = require("../services/player.service");
+const recorder = new recorder_service_1.RecorderService();
+const player = new player_service_1.PlayerService();
+const App = () => {
+    const [recording, setRecording] = (0, react_1.useState)(false);
+    const [basicCode, setBasicCode] = (0, react_1.useState)('');
+    const handleStartRecording = async () => {
+        setRecording(true);
+        await recorder.startRecording();
+    };
+    const handleStopRecording = async () => {
+        setRecording(false);
+        const code = await recorder.stopRecording();
+        setBasicCode(code);
+        // Save to file
+        const blob = new Blob([code], { type: 'text/plain' });
+        const url = URL.createObjectURL(blob);
+        const a = document.createElement('a');
+        a.href = url;
+        a.download = 'automation.bas';
+        a.click();
+    };
+    const handlePlayback = async () => {
+        try {
+            await player.executeBasicCode(basicCode);
+        }
+        catch (error) {
+            console.error('Playback error:', error);
+        }
+    };
+    return (react_1.default.createElement("div", { className: "p-4" },
+        react_1.default.createElement("h1", { className: "text-2xl font-bold mb-4" }, "BotDesktop Automation"),
+        react_1.default.createElement("div", { className: "space-x-4 mb-4" },
+            react_1.default.createElement("button", { className: `px-4 py-2 rounded ${recording ? 'bg-red-500' : 'bg-blue-500'} text-white`, onClick: recording ? handleStopRecording : handleStartRecording }, recording ? 'Stop Recording' : 'Start Recording'),
+            react_1.default.createElement("button", { className: "px-4 py-2 rounded bg-green-500 text-white", onClick: handlePlayback, disabled: !basicCode }, "Play Recording")),
+        react_1.default.createElement("div", { className: "mt-4" },
+            react_1.default.createElement("h2", { className: "text-xl font-bold mb-2" }, "Generated BASIC Code:"),
+            react_1.default.createElement("pre", { className: "bg-gray-100 p-2 rounded border" }, basicCode))));
+};
+exports.default = App;

dist/main/main.js

@@ -0,0 +1,72 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const electron_1 = require("electron");
+const path = __importStar(require("path"));
+const recorder_service_1 = require("../services/recorder.service");
+const player_service_1 = require("../services/player.service");
+const recorder = new recorder_service_1.RecorderService();
+const player = new player_service_1.PlayerService();
+function createWindow() {
+    const mainWindow = new electron_1.BrowserWindow({
+        width: 1200,
+        height: 800,
+        webPreferences: {
+            nodeIntegration: true,
+            contextIsolation: false,
+            preload: path.join(__dirname, '../preload/preload.js')
+        }
+    });
+    if (process.env.NODE_ENV === 'development') {
+        mainWindow.loadURL('http://localhost:8080');
+        mainWindow.webContents.openDevTools();
+    }
+    else {
+        mainWindow.loadFile(path.join(__dirname, '../../src/renderer/index.html'));
+    }
+}
+electron_1.app.whenReady().then(createWindow);
+electron_1.app.on('window-all-closed', () => {
+    if (process.platform !== 'darwin') {
+        electron_1.app.quit();
+    }
+});
+electron_1.app.on('activate', () => {
+    if (electron_1.BrowserWindow.getAllWindows().length === 0) {
+        createWindow();
+    }
+});
+electron_1.ipcMain.handle('mouse-event', recorder.mouseHandleEvent.bind(recorder));
+electron_1.ipcMain.handle('keyboard-event', recorder.keyboardHandleEvent.bind(recorder));
+electron_1.ipcMain.handle('screenshot-captured', recorder.screenshotHandleEvent.bind(recorder));
+electron_1.ipcMain.handle('start-recording', async () => {
+    await recorder.startRecording();
+});
+electron_1.ipcMain.handle('stop-recording', async () => {
+    return await recorder.stopRecording();
+});
+electron_1.ipcMain.handle('execute-basic-code', async (_, code) => {
+    await player.executeBasicCode(code);
+});

dist/preload/preload.js

@@ -0,0 +1 @@
+// Preload script goes here

dist/renderer/index.js

@@ -0,0 +1,10 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const react_1 = __importDefault(require("react"));
+const client_1 = __importDefault(require("react-dom/client"));
+const App_1 = __importDefault(require("../components/App"));
+client_1.default.createRoot(document.getElementById('root')).render(react_1.default.createElement(react_1.default.StrictMode, null,
+    react_1.default.createElement(App_1.default, null)));

dist/services/openai.service.js

@@ -0,0 +1,56 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAIService = void 0;
+const openai_1 = require("openai");
+const fs = __importStar(require("fs"));
+class OpenAIService {
+    constructor() {
+        this.client = new openai_1.AzureOpenAI({ dangerouslyAllowBrowser: true,
+            endpoint: process.env.AZURE_OPEN_AI_ENDPOINT || '',
+            deployment: process.env.AZURE_OPEN_AI_IMAGE_MODEL || '',
+            apiVersion: process.env.OPENAI_API_VERSION || '',
+            apiKey: process.env.AZURE_OPEN_AI_KEY || ''
+        });
+    }
+    async analyzeScreen(imagePath) {
+        const imageBuffer = fs.readFileSync(imagePath);
+        const base64Image = imageBuffer.toString('base64');
+        const response = await this.client.chat.completions.create({
+            model: process.env.AZURE_OPEN_AI_LLM_MODEL || '',
+            messages: [
+                {
+                    role: 'user',
+                    content: [
+                        { type: 'text', text: 'Analyze this screenshot and identify all interactive elements (buttons, text fields, etc). Return their locations and identifiers.' },
+                        { type: 'image_url', image_url: { url: `data:image/png;base64,${base64Image}` } }
+                    ],
+                },
+            ],
+        });
+        return JSON.parse(response.choices[0].message.content || '{}');
+    }
+}
+exports.OpenAIService = OpenAIService;

dist/services/player.service.js

@@ -0,0 +1,68 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PlayerService = void 0;
+const electron_1 = require("electron");
+const openai_service_1 = require("./openai.service");
+class PlayerService {
+    constructor() {
+        this.openAIService = new openai_service_1.OpenAIService();
+    }
+    async executeBasicCode(code) {
+        const lines = code.split('\n');
+        for (const line of lines) {
+            if (line.trim().startsWith('REM') || line.trim() === '')
+                continue;
+            const match = line.match(/^\d+\s+(\w+)\s+"([^"]+)"(?:\s+"([^"]+)")?/);
+            if (!match)
+                continue;
+            const [_, command, identifier, value] = match;
+            await this.executeCommand(command, identifier, value);
+            await new Promise(resolve => setTimeout(resolve, 500));
+        }
+    }
+    async executeCommand(command, identifier, value) {
+        // Capture current screen
+        const screenshotPath = await this.captureScreen();
+        const analysis = await this.openAIService.analyzeScreen(screenshotPath);
+        const element = analysis.elements.find(e => e.identifier === identifier);
+        if (!element)
+            throw new Error(`Element not found: ${identifier}`);
+        // Calculate center point of element
+        const centerX = element.bounds.x + element.bounds.width / 2;
+        const centerY = element.bounds.y + element.bounds.height / 2;
+        switch (command) {
+            case 'CLICK':
+                await this.simulateClick(centerX, centerY);
+                break;
+            case 'TYPE':
+                await this.simulateClick(centerX, centerY);
+                await this.simulateTyping(value || '');
+                break;
+        }
+    }
+    async captureScreen() {
+        return new Promise((resolve, reject) => {
+            electron_1.ipcMain.once('screen-captured', (_, screenshotPath) => {
+                resolve(screenshotPath);
+            });
+            electron_1.ipcMain.emit('capture-screen');
+        });
+    }
+    async simulateClick(x, y) {
+        return new Promise((resolve) => {
+            electron_1.ipcMain.once('click-completed', () => {
+                resolve();
+            });
+            electron_1.ipcMain.emit('simulate-click', { x, y });
+        });
+    }
+    async simulateTyping(text) {
+        return new Promise((resolve) => {
+            electron_1.ipcMain.once('typing-completed', () => {
+                resolve();
+            });
+            electron_1.ipcMain.emit('simulate-typing', { text });
+        });
+    }
+}
+exports.PlayerService = PlayerService;

dist/services/recorder.service.js

@@ -0,0 +1,93 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecorderService = void 0;
+const electron_1 = require("electron");
+const dotenv_1 = __importDefault(require("dotenv"));
+dotenv_1.default.config();
+const openai_service_1 = require("./openai.service");
+class RecorderService {
+    constructor() {
+        this.events = [];
+        this.recording = false;
+        this.currentScreenshot = '';
+        this.openAIService = new openai_service_1.OpenAIService();
+    }
+    async startRecording() {
+        this.recording = true;
+        this.events = [];
+        this.requestScreenshot();
+    }
+    stopRecording() {
+        this.recording = false;
+        return this.generateBasicCode();
+    }
+    requestScreenshot() {
+        // Notify renderer process to capture a screenshot
+        const allWebContents = electron_1.screen.getAllDisplays();
+        allWebContents.forEach((webContents) => {
+            //@ts-ignores
+            webContents.send('request-screenshot');
+        });
+    }
+    async screenshotHandleEvent(_, screenshot) {
+        this.currentScreenshot = screenshot; // Store the screenshot as a base64 image
+    }
+    async mouseHandleEvent(_, event) {
+        if (!this.recording)
+            return;
+        const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot);
+        const element = this.findElementAtPosition(analysis, event.x, event.y);
+        if (element) {
+            this.events.push({
+                type: 'click',
+                identifier: element.identifier,
+                timestamp: Date.now(),
+            });
+        }
+    }
+    async keyboardHandleEvent(_, event) {
+        if (!this.recording)
+            return;
+        const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot);
+        const focusedElement = this.findFocusedElement(analysis);
+        if (focusedElement) {
+            this.events.push({
+                type: 'type',
+                identifier: focusedElement.identifier,
+                value: event.key,
+                timestamp: Date.now(),
+            });
+        }
+    }
+    findElementAtPosition(analysis, x, y) {
+        return analysis.elements.find((element) => {
+            const bounds = element.bounds;
+            return x >= bounds.x && x <= bounds.x + bounds.width && y >= bounds.y && y <= bounds.y + bounds.height;
+        });
+    }
+    findFocusedElement(analysis) {
+        //@ts-ignore
+        return analysis.elements.find((element) => element.focused);
+    }
+    generateBasicCode() {
+        let basicCode = '10 REM BotDesktop Automation Script\n';
+        let lineNumber = 20;
+        for (const event of this.events) {
+            switch (event.type) {
+                case 'click':
+                    basicCode += `${lineNumber} CLICK "${event.identifier}"\n`;
+                    break;
+                case 'type':
+                    basicCode += `${lineNumber} TYPE "${event.identifier}" "${event.value}"\n`;
+                    break;
+            }
+            lineNumber += 10;
+        }
+        basicCode += `${lineNumber} END\n`;
+        return basicCode;
+    }
+}
+exports.RecorderService = RecorderService;

dist/services/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

dist/tests/services.test.js

@@ -0,0 +1 @@
+// Tests for services

electron-builder.json

@@ -0,0 +1,20 @@
+{
+  "appId": "com.botdesktop.app",
+  "directories": {
+    "output": "release/"
+  },
+  "files": [
+    "dist/**/*",
+    "node_modules/**/*",
+    "package.json"
+  ],
+  "mac": {
+    "target": ["dmg"]
+  },
+  "win": {
+    "target": ["nsis"]
+  },
+  "linux": {
+    "target": ["AppImage"]
+  }
+}

package.json

@@ -0,0 +1,35 @@
+{
+  "name": "bot-desktop",
+  "version": "1.0.0",
+  "description": "AI-powered desktop automation tool",
+  "main": "dist/main/main.js",
+  "scripts": {
+    "start": "electron .",
+    "dev": "concurrently \"webpack serve --mode development\" \"tsc -w -p tsconfig.electron.json\" \"electron .\"",
+    "build": "webpack --mode production && tsc -p tsconfig.electron.json && electron-builder",
+    "test": "vitest"
+  },
+  "dependencies": {
+    "dotenv": "^16.4.5",
+    "node-global-key-listener": "^0.3.0",
+    "node-mouse": "^0.0.2",
+    "@types/node": "^20.0.0",
+    "@types/react": "^18.0.0",
+    "@types/react-dom": "^18.0.0",
+    "electron": "^28.0.0",
+    "openai": "^4.28.0",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "typescript": "^5.0.0"
+  },
+  "devDependencies": {
+    "concurrently": "^8.2.2",
+    "electron-builder": "^24.9.1",
+    "html-webpack-plugin": "^5.6.0",
+    "ts-loader": "^9.5.1",
+    "vitest": "^1.2.1",
+    "webpack": "^5.89.0",
+    "webpack-cli": "^5.1.4",
+    "webpack-dev-server": "^4.15.1"
+  }
+}

src/components/App.tsx

@@ -0,0 +1,67 @@
+import React, { useState } from 'react';
+import { RecorderService } from '../services/recorder.service';
+import { PlayerService } from '../services/player.service';
+
+const recorder = new RecorderService();
+const player = new PlayerService();
+
+const App: React.FC = () => {
+  const [recording, setRecording] = useState(false);
+  const [basicCode, setBasicCode] = useState('');
+
+  const handleStartRecording = async () => {
+    setRecording(true);
+    await recorder.startRecording();
+  };
+
+  const handleStopRecording = async () => {
+    setRecording(false);
+    const code = await recorder.stopRecording();
+    setBasicCode(code);
+    // Save to file
+    const blob = new Blob([code], { type: 'text/plain' });
+    const url = URL.createObjectURL(blob);
+    const a = document.createElement('a');
+    a.href = url;
+    a.download = 'automation.bas';
+    a.click();
+  };
+
+  const handlePlayback = async () => {
+    try {
+      await player.executeBasicCode(basicCode);
+    } catch (error) {
+      console.error('Playback error:', error);
+    }
+  };
+
+  return (
+    <div className="p-4">
+      <h1 className="text-2xl font-bold mb-4">BotDesktop Automation</h1>
+      
+      <div className="space-x-4 mb-4">
+        <button
+          className={`px-4 py-2 rounded ${recording ? 'bg-red-500' : 'bg-blue-500'} text-white`}
+          onClick={recording ? handleStopRecording : handleStartRecording}
+        >
+          {recording ? 'Stop Recording' : 'Start Recording'}
+        </button>
+        
+        <button
+          className="px-4 py-2 rounded bg-green-500 text-white"
+          onClick={handlePlayback}
+          disabled={!basicCode}
+        >
+          Play Recording
+        </button>
+      </div>
+
+      <div className="mt-4">
+        <h2 className="text-xl font-bold mb-2">Generated BASIC Code:</h2>
+        <pre className="bg-gray-100 p-2 rounded border">{basicCode}</pre>
+      </div>
+    </div>
+  );
+};
+
+export default App;

src/main/main.ts

@@ -0,0 +1,58 @@
+import { app, BrowserWindow, ipcMain } from 'electron';
+import * as path from 'path';
+
+import { RecorderService } from '../services/recorder.service';
+import { PlayerService } from '../services/player.service';
+
+const recorder = new RecorderService();
+const player = new PlayerService();
+
+function createWindow() {
+  const mainWindow = new BrowserWindow({
+    width: 1200,
+    height: 800,
+    webPreferences: {
+      nodeIntegration: true,
+      contextIsolation: false,
+      preload: path.join(__dirname, '../preload/preload.js')
+    }
+  });
+
+  if (process.env.NODE_ENV === 'development') {
+    mainWindow.loadURL('http://localhost:8080');
+    mainWindow.webContents.openDevTools();
+  } else {
+    mainWindow.loadFile(path.join(__dirname, '../../src/renderer/index.html'));
+  }
+}
+
+app.whenReady().then(createWindow);
+
+app.on('window-all-closed', () => {
+  if (process.platform !== 'darwin') {
+    app.quit();
+  }
+});
+
+app.on('activate', () => {
+  if (BrowserWindow.getAllWindows().length === 0) {
+    createWindow();
+  }
+});
+
+ipcMain.handle('mouse-event', recorder.mouseHandleEvent.bind(recorder));
+ipcMain.handle('keyboard-event', recorder.keyboardHandleEvent.bind(recorder));
+ipcMain.handle('screenshot-captured', recorder.screenshotHandleEvent.bind(recorder));
+
+
+ipcMain.handle('start-recording', async () => {
+  await recorder.startRecording();
+});
+
+ipcMain.handle('stop-recording', async () => {
+  return await recorder.stopRecording();
+});
+
+ipcMain.handle('execute-basic-code', async (_, code: string) => {
+  await player.executeBasicCode(code);
+});

src/preload/preload.ts

@@ -0,0 +1 @@
+// Preload script goes here

src/renderer/index.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>BotDesktop</title>
+    <script src="https://cdn.tailwindcss.com"></script>
+</head>
+<body>
+    <div id="root"></div>
+</body>
+</html>

src/renderer/index.tsx

@@ -0,0 +1,14 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+
+ 
+
+import App from '../components/App';
+
+ReactDOM.createRoot(
+  document.getElementById('root') as HTMLElement
+).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+);

src/services/openai.service.ts

@@ -0,0 +1,36 @@
+import { AzureOpenAI } from 'openai';
+import * as fs from 'fs';
+import { ScreenAnalysis } from './types';
+
+export class OpenAIService {
+  private client: AzureOpenAI;
+
+  constructor() {
+    this.client = new AzureOpenAI({ dangerouslyAllowBrowser: true,
+      endpoint: process.env.AZURE_OPEN_AI_ENDPOINT || '',
+      deployment: process.env.AZURE_OPEN_AI_IMAGE_MODEL || '',
+      apiVersion: process.env.OPENAI_API_VERSION || '',
+      apiKey: process.env.AZURE_OPEN_AI_KEY || ''
+    });
+  }
+
+  async analyzeScreen(imagePath: string): Promise<ScreenAnalysis> {
+    const imageBuffer = fs.readFileSync(imagePath);
+    const base64Image = imageBuffer.toString('base64');
+
+    const response = await this.client.chat.completions.create({
+      model: process.env.AZURE_OPEN_AI_LLM_MODEL || '',
+      messages: [
+        {
+          role: 'user',
+          content: [
+            { type: 'text', text: 'Analyze this screenshot and identify all interactive elements (buttons, text fields, etc). Return their locations and identifiers.' },
+            { type: 'image_url', image_url: { url: `data:image/png;base64,${base64Image}` } }
+          ],
+        },
+      ],
+    });
+
+    return JSON.parse(response.choices[0].message.content || '{}');
+  }
+}

src/services/player.service.ts

@@ -0,0 +1,80 @@
+import { ipcMain } from 'electron';
+import { AutomationEvent, ScreenAnalysis } from './types';
+import { OpenAIService } from './openai.service';
+
+export class PlayerService {
+  private openAIService: OpenAIService;
+
+  constructor() {
+    this.openAIService = new OpenAIService();
+  }
+
+  async executeBasicCode(code: string) {
+    const lines = code.split('\n');
+    
+    for (const line of lines) {
+      if (line.trim().startsWith('REM') || line.trim() === '') continue;
+      
+      const match = line.match(/^\d+\s+(\w+)\s+"([^"]+)"(?:\s+"([^"]+)")?/);
+      if (!match) continue;
+
+      const [_, command, identifier, value] = match;
+      await this.executeCommand(command, identifier, value);
+      await new Promise(resolve => setTimeout(resolve, 500));
+    }
+  }
+
+  private async executeCommand(command: string, identifier: string, value?: string) {
+    // Capture current screen
+    const screenshotPath = await this.captureScreen();
+    
+    const analysis = await this.openAIService.analyzeScreen(screenshotPath);
+    const element = analysis.elements.find(e => e.identifier === identifier);
+    
+    if (!element) throw new Error(`Element not found: ${identifier}`);
+
+    // Calculate center point of element
+    const centerX = element.bounds.x + element.bounds.width/2;
+    const centerY = element.bounds.y + element.bounds.height/2;
+
+    switch (command) {
+      case 'CLICK':
+        await this.simulateClick(centerX, centerY);
+        break;
+      case 'TYPE':
+        await this.simulateClick(centerX, centerY);
+        await this.simulateTyping(value || '');
+        break;
+    }
+  }
+
+  private async captureScreen(): Promise<string> {
+    return new Promise((resolve, reject) => {
+      ipcMain.once('screen-captured', (_, screenshotPath) => {
+        resolve(screenshotPath);
+      });
+
+      ipcMain.emit('capture-screen');
+    });
+  }
+
+  private async simulateClick(x: number, y: number): Promise<void> {
+    return new Promise((resolve) => {
+      ipcMain.once('click-completed', () => {
+        resolve();
+      });
+
+      ipcMain.emit('simulate-click', { x, y });
+    });
+  }
+
+  private async simulateTyping(text: string): Promise<void> {
+    return new Promise((resolve) => {
+      ipcMain.once('typing-completed', () => {
+        resolve();
+      });
+
+      ipcMain.emit('simulate-typing', { text });
+    });
+  }
+}

src/services/recorder.service.ts

@@ -0,0 +1,103 @@
+import { screen, ipcMain } from 'electron';
+import { AutomationEvent, ScreenAnalysis } from './types';
+import dotenv from 'dotenv';
+dotenv.config();
+import { OpenAIService } from './openai.service';
+
+export class RecorderService {
+  private events: AutomationEvent[] = [];
+  private recording: boolean = false;
+  private openAIService: OpenAIService;
+  private currentScreenshot: string = '';
+
+  constructor() {
+    this.openAIService = new OpenAIService();
+  }
+
+  public async startRecording() {
+    this.recording = true;
+    this.events = [];
+    this.requestScreenshot();
+  }
+
+  public stopRecording(): string {
+    this.recording = false;
+    return this.generateBasicCode();
+  }
+
+  private requestScreenshot() {
+    // Notify renderer process to capture a screenshot
+    const allWebContents = screen.getAllDisplays();
+    allWebContents.forEach((webContents) => {
+      //@ts-ignores
+      webContents.send('request-screenshot');
+    });
+  }
+
+  public async screenshotHandleEvent (_: any, screenshot: string) {
+    this.currentScreenshot = screenshot; // Store the screenshot as a base64 image
+  }
+
+  public async mouseHandleEvent(_: any, event: any) {
+    if (!this.recording) return;
+
+    const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot);
+    const element = this.findElementAtPosition(analysis, event.x, event.y);
+
+    if (element) {
+      this.events.push({
+        type: 'click',
+        identifier: element.identifier,
+        timestamp: Date.now(),
+      });
+    }
+  }
+
+  public async keyboardHandleEvent(_: any, event: any) {
+    if (!this.recording) return;
+
+    const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot);
+    const focusedElement = this.findFocusedElement(analysis);
+
+    if (focusedElement) {
+      this.events.push({
+        type: 'type',
+        identifier: focusedElement.identifier,
+        value: event.key,
+        timestamp: Date.now(),
+      });
+    }
+  }
+
+  private findElementAtPosition(analysis: ScreenAnalysis, x: number, y: number) {
+    return analysis.elements.find((element) => {
+      const bounds = element.bounds;
+      return x >= bounds.x && x <= bounds.x + bounds.width && y >= bounds.y && y <= bounds.y + bounds.height;
+    });
+  }
+
+  private findFocusedElement(analysis: ScreenAnalysis) {
+    //@ts-ignore
+    return analysis.elements.find((element) => element.focused);
+  }
+
+  private generateBasicCode(): string {
+    let basicCode = '10 REM BotDesktop Automation Script\n';
+    let lineNumber = 20;
+
+    for (const event of this.events) {
+      switch (event.type) {
+        case 'click':
+          basicCode += `${lineNumber} CLICK "${event.identifier}"\n`;
+          break;
+        case 'type':
+          basicCode += `${lineNumber} TYPE "${event.identifier}" "${event.value}"\n`;
+          break;
+      }
+      lineNumber += 10;
+    }
+
+    basicCode += `${lineNumber} END\n`;
+    return basicCode;
+  }
+}

src/services/types.ts

@@ -0,0 +1,15 @@
+export interface AutomationEvent {
+  type: 'click' | 'type' | 'move';
+  identifier: string;
+  value?: string;
+  timestamp: number;
+}
+
+export interface ScreenAnalysis {
+  elements: {
+    identifier: string;
+    type: string;
+    bounds: { x: number, y: number, width: number, height: number };
+    value?: string;
+  }[];
+}

src/tests/services.test.ts

@@ -0,0 +1 @@
+// Tests for services

tsconfig.electron.json

@@ -0,0 +1,7 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "module": "CommonJS",
+    "outDir": "dist"
+  }
+}

tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "CommonJS",
+    "lib": ["DOM", "ES2020"],
+    "jsx": "react",
+    "strict": false,
+    "noImplicitAny": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "outDir": "dist",
+    "rootDir": "src",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["src/*"]
+    }
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules"]
+}

webpack.config.js

@@ -0,0 +1,28 @@
+const path = require('path');
+const HtmlWebpackPlugin = require('html-webpack-plugin');
+
+module.exports = {
+  entry: './src/renderer/index.tsx',
+  target: 'electron-renderer',
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        use: 'ts-loader',
+        exclude: /node_modules/,
+      },
+    ],
+  },
+  resolve: {
+    extensions: ['.tsx', '.ts', '.js'],
+  },
+  output: {
+    filename: 'renderer.js',
+    path: path.resolve(__dirname, 'dist/renderer'),
+  },
+  plugins: [
+    new HtmlWebpackPlugin({
+      template: './src/renderer/index.html'
+    }),
+  ],
+};