GeneralBots/botui
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

new(all): Initial import.

Browse source
This commit is contained in:
me@rodrigorodriguez.com 2024-10-26 21:21:51 -03:00
parent 5ebde5b646
commit c6f0933bfd
17 changed files with 542 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
gencode.sh → cexp.sh
View file

337
cimp.sh Executable file
View file

@ -0,0 +1,337 @@
#!/bin/bash
# Create project directories
mkdir -p ./src/preload
mkdir -p ./src/renderer
mkdir -p ./src/services
# Create preload.ts file
cat <<EOL > ./src/preload/preload.ts
// File: ./src/preload/preload.ts
const { ipcRenderer } = require('electron');
//@ts-nocheck
(window as any).myApi = {
//@ts-nocheck
sendMessage: (message: any) => {
console.log('preload.sendMessage', { message });
ipcRenderer.send('message-from-renderer', message);
},
//@ts-nocheck
receiveMessage: (callback: any) => {
console.log('preload.receiveMessage', { callback });
ipcRenderer.on('message-from-main', (event, arg) => callback(arg));
},
};
EOL
# Create index.tsx file
cat <<EOL > ./src/renderer/index.tsx
// File: ./src/renderer/index.tsx
import React from 'react';
import ReactDOM from 'react-dom/client';
import App from '../components/App';
ReactDOM.createRoot(
document.getElementById('root') as HTMLElement
).render(
<React.StrictMode>
<App />
</React.StrictMode>
);
EOL
# Create player.service.ts file
cat <<EOL > ./src/services/player.service.ts
// File: ./src/services/player.service.ts
import { ipcMain } from 'electron';
import { AutomationEvent, ScreenAnalysis } from './types';
import { OpenAIService } from './openai.service';
export class PlayerService {
private openAIService: OpenAIService;
constructor() {
console.log('PlayerService.constructor', {});
this.openAIService = new OpenAIService();
}
async executeBasicCode(code: string) {
console.log('PlayerService.executeBasicCode', { code });
const lines = code.split('\\n');
for (const line of lines) {
if (line.trim().startsWith('REM') || line.trim() === '') continue;
const match = line.match(/^\\d+\\s+(\\w+)\\s+"([^"]+)"(?:\\s+"([^"]+)")?/);
if (!match) continue;
const [_, command, identifier, value] = match;
await this.executeCommand(command, identifier, value);
await new Promise(resolve => setTimeout(resolve, 500));
}
}
private async executeCommand(command: string, identifier: string, value?: string) {
console.log('PlayerService.executeCommand', { command, identifier, value });
const screenshotPath = await this.captureScreen();
const analysis = await this.openAIService.analyzeScreen(screenshotPath);
const element = analysis.elements.find(e => e.identifier === identifier);
if (!element) throw new Error(\`Element not found: \${identifier}\`);
const centerX = element.bounds.x + element.bounds.width / 2;
const centerY = element.bounds.y + element.bounds.height / 2;
switch (command) {
case 'CLICK':
await this.simulateClick(centerX, centerY);
break;
case 'TYPE':
await this.simulateClick(centerX, centerY);
await this.simulateTyping(value || '');
break;
}
}
private async captureScreen(): Promise<string> {
console.log('PlayerService.captureScreen', {});
return new Promise((resolve, reject) => {
ipcMain.once('screen-captured', (_, screenshotPath) => {
resolve(screenshotPath);
});
ipcMain.emit('capture-screen');
});
}
private async simulateClick(x: number, y: number): Promise<void> {
console.log('PlayerService.simulateClick', { x, y });
return new Promise((resolve) => {
ipcMain.once('click-completed', () => {
resolve();
});
ipcMain.emit('simulate-click', { x, y });
});
}
private async simulateTyping(text: string): Promise<void> {
console.log('PlayerService.simulateTyping', { text });
return new Promise((resolve) => {
ipcMain.once('typing-completed', () => {
resolve();
});
ipcMain.emit('simulate-typing', { text });
});
}
}
EOL
# Create types.ts file
cat <<EOL > ./src/services/types.ts
// File: ./src/services/types.ts
export interface AutomationAction {
type: 'click' | 'type' | 'move';
identifier: string;
value?: string;
confidence: number;
bounds: {
x: number;
y: number;
width: number;
height: number;
};
}
export interface AutomationEvent {
type: 'click' | 'type' | 'move';
identifier: string;
value?: string;
timestamp: number;
narration: string;
}
export interface WhisperResponse {
text: string;
segments: any;
}
export interface ScreenContext {
screenshot: string;
transcription: string;
cursorPosition: { x: number, y: number };
}
export interface ScreenAnalysis {
timestamp: number,
elements: {
identifier: string;
type: string;
bounds: { x: number; y: number; width: number; height: number };
value?: string;
}[];
}
EOL
# Create recorder.service.ts file
cat <<EOL > ./src/services/recorder.service.ts
// File: ./src/services/recorder.service.ts
const { ipcRenderer } = require('electron'); // Require ipcRender
import { AutomationEvent, ScreenAnalysis, WhisperResponse } from '../services/types';
import { OpenAIService } from '../services/openai.service';
import * as path from 'path';
import * as fs from 'fs';
export class RecorderService {
private events: AutomationEvent[] = [];
private recording: boolean = false;
private openAIService: OpenAIService;
private currentScreenshot: string = '';
private lastTranscription: string = '';
private recordingProcess: any = null;
private tempDir: string;
private currentAudioFile: string = '';
private silenceTimer: NodeJS.Timeout | null = null;
private isProcessingAudio: boolean = false;
constructor() {
console.log('RecorderService.constructor', {});
this.openAIService = new OpenAIService();
this.tempDir = path.join(process.cwd(), 'temp_recordings');
if (!fs.existsSync(this.tempDir)) {
fs.mkdirSync(this.tempDir, { recursive: true });
}
}
public async startRecording() {
console.log('RecorderService.startRecording', {});
try {
this.recording = true;
this.events = [];
await this.setupAudioRecording();
await this.requestScreenshot();
ipcRenderer.on('keyboard-event', this.keyboardHandleEvent);
} catch (error) {
console.error('Failed to start recording:', error);
this.recording = false;
throw error;
}
}
private async setupAudioRecording() {
console.log('RecorderService.setupAudioRecording', {});
try {
ipcRenderer.on('audio-level', this.handleAudioLevel);
ipcRenderer.on('audio-chunk', this.handleAudioChunk);
} catch (error) {
console.error('Error setting up audio recording:', error);
throw new Error(\`Failed to setup audio recording: \${error.message}\`);
}
}
private handleAudioLevel = async (_: any, level: number) => {
console.log('RecorderService.handleAudioLevel', { level });
if (!this.recording) return;
const SILENCE_THRESHOLD = 0.01;
const SILENCE_DURATION = 1000;
if (level < SILENCE_THRESHOLD) {
if (!this.silenceTimer && !this.isProcessingAudio) {
this.silenceTimer = setTimeout(async () => {
if (this.recording) {
await this.processSilence();
}
}, SILENCE_DURATION);
}
} else {
if (this.silenceTimer) {
clearTimeout(this.silenceTimer);
this.silenceTimer = null;
}
}
}
private handleAudioChunk = async (_: any, chunk: Buffer) => {
console.log('RecorderService.handleAudioChunk', { chunk });
if (!this.recording) return;
try {
const audioFilePath = path.join(this.tempDir, \`audio-\${Date.now()}.wav\`);
fs.writeFileSync(audioFilePath, chunk);
if (this.silenceTimer) {
clearTimeout(this.silenceTimer);
this.silenceTimer = null;
await this.processAudioFile(audioFilePath);
}
} catch (error) {
console.error('Error handling audio chunk:', error);
}
};
private async processSilence() {
console.log('RecorderService.processSilence', {});
if (this.isProcessingAudio) return;
this.isProcessingAudio = true;
try {
const audioFilePath = await ipcRenderer.invoke('save-audio-chunk');
if (audioFilePath) {
this.currentAudioFile = audioFilePath;
await this.processAudioFile(audioFilePath);
await this.requestScreenshot();
}
} catch (error) {
console.error('Error processing silence:', error);
} finally {
this.isProcessingAudio = false;
}
}
private async processAudioFile(audioFilePath: string) {
console.log('RecorderService.processAudioFile', { audioFilePath });
const transcription = await this.openAIService.transcribeAudio(audioFilePath);
this.lastTranscription = transcription;
await this.requestScreenshot();
}
private async requestScreenshot() {
console.log('RecorderService.requestScreenshot', {});
await ipcRenderer.invoke('request-screenshot');
}
private keyboardHandleEvent = async (_: any, event: any) => {
console.log('RecorderService.keyboardHandleEvent', { event });
if (!this.recording) return;
const automationEvent: AutomationEvent = {
type: 'keyboard',
identifier: event.key,
timestamp: Date.now(),
narration: this.lastTranscription,
};
this.events.push(automationEvent);
};
public async stopRecording() {
console.log('RecorderService.stopRecording', {});
try {
this.recording = false;
ipcRenderer.removeListener('keyboard-event', this.keyboardHandleEvent);
await ipcRenderer.invoke('stop-audio-recording');
} catch (error) {
console.error('Failed to stop recording:', error);
}
}
}
EOL

20
dist/main/main.js vendored
View file

@ -24,9 +24,9 @@ var __importStar = (this && this.__importStar) || function (mod) {
};
Object.defineProperty(exports, "__esModule", { value: true });
require('dotenv').config();
require('electron-require');
const electron_1 = require("electron");
const path = __importStar(require("path"));
// In main.ts
const electron_2 = require("electron");
const recorder_service_1 = require("../services/recorder.service");
const player_service_1 = require("../services/player.service");
@ -37,7 +37,9 @@ function createWindow() {
width: 1200,
height: 800,
webPreferences: {
nodeIntegrationInWorker: true,
nodeIntegration: true,
nodeIntegrationInSubFrames: true,
contextIsolation: false,
preload: path.join(__dirname, '../preload/preload.js')
}
@ -64,17 +66,28 @@ electron_1.app.on('activate', () => {
electron_1.ipcMain.handle('mouse-event', recorder.mouseHandleEvent.bind(recorder));
electron_1.ipcMain.handle('keyboard-event', recorder.keyboardHandleEvent.bind(recorder));
electron_1.ipcMain.handle('screenshot-captured', recorder.screenshotHandleEvent.bind(recorder));
// Handler to capture the entire screen
electron_1.ipcMain.handle('get-screenshot', async () => {
console.log('get-screenshot called');
const sources = await electron_1.desktopCapturer.getSources({ types: ['screen'] });
const screenSource = sources[0]; // Get the first screen source
const { thumbnail } = screenSource; // Thumbnail is a native image
return thumbnail.toPNG(); // Return the screenshot as PNG buffer
});
electron_1.ipcMain.handle('start-recording', async () => {
console.log('start-recording called');
await recorder.startRecording();
});
electron_1.ipcMain.handle('stop-recording', async () => {
console.log('stop-recording called');
return await recorder.stopRecording();
});
electron_1.ipcMain.handle('execute-basic-code', async (_, code) => {
console.log('execute-basic-code called with:', code);
await player.executeBasicCode(code);
});
// Add microphone permission check for macOS
electron_1.ipcMain.handle('check-microphone-permission', async () => {
console.log('check-microphone-permission called');
if (process.platform === 'darwin') {
const status = await electron_2.systemPreferences.getMediaAccessStatus('microphone');
if (status !== 'granted') {
@ -83,8 +96,7 @@ electron_1.ipcMain.handle('check-microphone-permission', async () => {
}
return true;
}
// On Windows/Linux, permissions are handled by the OS
return true;
return true; // On Windows/Linux, permissions are handled by the OS
});
// Enable required permissions
electron_1.app.commandLine.appendSwitch('enable-speech-dispatcher');

12
dist/preload/preload.js vendored
View file

@ -0,0 +1,12 @@
const { ipcRenderer } = require('electron');
//@ts-nocheck
window.myApi = {
sendMessage: (message) => {
console.log('[preload] sendMessage called with:', message);
return ipcRenderer.send('message-from-renderer', message);
},
receiveMessage: (callback) => {
console.log('[preload] receiveMessage registered with callback');
return ipcRenderer.on('message-from-main', (event, arg) => callback(arg));
},
};

1
dist/renderer/index.js vendored
View file

@ -6,5 +6,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
const react_1 = __importDefault(require("react"));
const client_1 = __importDefault(require("react-dom/client"));
const App_1 = __importDefault(require("../components/App"));
console.log('[renderer] Initializing React app');
client_1.default.createRoot(document.getElementById('root')).render(react_1.default.createElement(react_1.default.StrictMode, null,
react_1.default.createElement(App_1.default, null)));

15
dist/services/player.service.js vendored
View file

@ -5,9 +5,11 @@ const electron_1 = require("electron");
const openai_service_1 = require("./openai.service");
class PlayerService {
constructor() {
console.log('[PlayerService] Initializing');
this.openAIService = new openai_service_1.OpenAIService();
}
async executeBasicCode(code) {
console.log('[PlayerService] executeBasicCode called with:', code);
const lines = code.split('\n');
for (const line of lines) {
if (line.trim().startsWith('REM') || line.trim() === '')
@ -16,49 +18,58 @@ class PlayerService {
if (!match)
continue;
const [_, command, identifier, value] = match;
console.log('[PlayerService] Executing command:', { command, identifier, value });
await this.executeCommand(command, identifier, value);
await new Promise(resolve => setTimeout(resolve, 500));
}
}
async executeCommand(command, identifier, value) {
// Capture current screen
console.log('[PlayerService] executeCommand called with:', { command, identifier, value });
const screenshotPath = await this.captureScreen();
console.log('[PlayerService] Screen captured at:', screenshotPath);
const analysis = await this.openAIService.analyzeScreen(screenshotPath);
const element = analysis.elements.find(e => e.identifier === identifier);
if (!element)
throw new Error(`Element not found: ${identifier}`);
// Calculate center point of element
const centerX = element.bounds.x + element.bounds.width / 2;
const centerY = element.bounds.y + element.bounds.height / 2;
switch (command) {
case 'CLICK':
console.log('[PlayerService] Simulating click at:', { centerX, centerY });
await this.simulateClick(centerX, centerY);
break;
case 'TYPE':
console.log('[PlayerService] Simulating type:', { centerX, centerY, value });
await this.simulateClick(centerX, centerY);
await this.simulateTyping(value || '');
break;
}
}
async captureScreen() {
console.log('[PlayerService] captureScreen called');
return new Promise((resolve, reject) => {
electron_1.ipcMain.once('screen-captured', (_, screenshotPath) => {
console.log('[PlayerService] Screen captured event received:', screenshotPath);
resolve(screenshotPath);
});
electron_1.ipcMain.emit('capture-screen');
});
}
async simulateClick(x, y) {
console.log('[PlayerService] simulateClick called with:', { x, y });
return new Promise((resolve) => {
electron_1.ipcMain.once('click-completed', () => {
console.log('[PlayerService] Click completed');
resolve();
});
electron_1.ipcMain.emit('simulate-click', { x, y });
});
}
async simulateTyping(text) {
console.log('[PlayerService] simulateTyping called with:', text);
return new Promise((resolve) => {
electron_1.ipcMain.once('typing-completed', () => {
console.log('[PlayerService] Typing completed');
resolve();
});
electron_1.ipcMain.emit('simulate-typing', { text });

66
dist/services/recorder.service.js vendored
View file

@ -26,7 +26,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
exports.RecorderService = void 0;
const electron_1 = require("electron");
const openai_service_1 = require("../services/openai.service");
const _ = require('lodash');
const path = __importStar(require("path"));
const fs = __importStar(require("fs"));
class RecorderService {
@ -35,17 +34,18 @@ class RecorderService {
this.recording = false;
this.currentScreenshot = '';
this.lastTranscription = '';
this.recordingProcess = null;
this.currentAudioFile = '';
this.silenceTimer = null;
this.isProcessingAudio = false;
this.handleAudioLevel = _.debounce(async (_, level) => {
this.handleAudioLevel = async (_, level) => {
console.log('RecorderService.handleAudioLevel()', { level });
if (!this.recording)
return;
const SILENCE_THRESHOLD = 0.01;
const SILENCE_DURATION = 1000;
if (level < SILENCE_THRESHOLD) {
if (!this.silenceTimer && !this.isProcessingAudio) {
console.log('RecorderService.handleAudioLevel() - Setting silence timer');
this.silenceTimer = setTimeout(async () => {
if (this.recording) {
await this.processSilence();
@ -55,12 +55,14 @@ class RecorderService {
}
else {
if (this.silenceTimer) {
console.log('RecorderService.handleAudioLevel() - Clearing silence timer');
clearTimeout(this.silenceTimer);
this.silenceTimer = null;
}
}
}, 100);
};
this.handleAudioChunk = async (_, chunk) => {
console.log('RecorderService.handleAudioChunk()', { chunkSize: chunk.length });
if (!this.recording)
return;
try {
@ -73,9 +75,10 @@ class RecorderService {
}
}
catch (error) {
console.error('Error handling audio chunk:', error);
console.error('RecorderService.handleAudioChunk() error:', error);
}
};
console.log('RecorderService.constructor()');
this.openAIService = new openai_service_1.OpenAIService();
this.tempDir = path.join(process.cwd(), 'temp_recordings');
if (!fs.existsSync(this.tempDir)) {
@ -83,36 +86,39 @@ class RecorderService {
}
}
async startRecording() {
console.log('RecorderService.startRecording()');
try {
this.recording = true;
this.events = [];
await this.setupAudioRecording();
await this.requestScreenshot();
electron_1.ipcRenderer.on('keyboard-event', this.keyboardHandleEvent); // Listen for keyboard events
electron_1.ipcRenderer.on('keyboard-event', this.keyboardHandleEvent);
}
catch (error) {
console.error('Failed to start recording:', error);
console.error('RecorderService.startRecording() error:', error);
this.recording = false;
throw error;
}
}
async setupAudioRecording() {
console.log('RecorderService.setupAudioRecording()');
try {
this.recordingProcess = await electron_1.ipcRenderer.invoke('start-audio-recording');
electron_1.ipcRenderer.on('audio-level', this.handleAudioLevel);
electron_1.ipcRenderer.on('audio-chunk', this.handleAudioChunk);
}
catch (error) {
console.error('Error setting up audio recording:', error);
console.error('RecorderService.setupAudioRecording() error:', error);
throw new Error(`Failed to setup audio recording: ${error.message}`);
}
}
async processSilence() {
console.log('RecorderService.processSilence()');
if (this.isProcessingAudio)
return;
this.isProcessingAudio = true;
try {
const audioFilePath = await electron_1.ipcRenderer.invoke('save-audio-chunk');
console.log('RecorderService.processSilence() - Audio saved to:', audioFilePath);
if (audioFilePath) {
this.currentAudioFile = audioFilePath;
await this.processAudioFile(audioFilePath);
@ -120,32 +126,38 @@ class RecorderService {
}
}
catch (error) {
console.error('Error processing silence:', error);
console.error('RecorderService.processSilence() error:', error);
}
finally {
this.isProcessingAudio = false;
}
}
async processAudioFile(audioFilePath) {
console.log('RecorderService.processAudioFile()', { audioFilePath });
try {
const audioBuffer = fs.readFileSync(audioFilePath);
const transcription = await this.openAIService.transcribeAudio(new Blob([audioBuffer], { type: 'audio/wav' }));
console.log('RecorderService.processAudioFile() - Transcription:', transcription);
if (transcription.text.trim()) {
await this.processTranscription(transcription);
}
fs.unlinkSync(audioFilePath);
}
catch (error) {
console.error('Error processing audio file:', error);
console.error('RecorderService.processAudioFile() error:', error);
}
}
async processTranscription(transcription) {
console.log('RecorderService.processTranscription()', { transcription });
this.lastTranscription = transcription.text;
const cursorPosition = await electron_1.ipcRenderer.invoke('get-cursor-position');
console.log('RecorderService.processTranscription() - Cursor position:', cursorPosition);
const analysis = await this.openAIService.analyzeScreenWithContext({
screenshot: this.currentScreenshot,
transcription: this.lastTranscription,
cursorPosition: await electron_1.ipcRenderer.invoke('get-cursor-position')
cursorPosition
});
console.log('RecorderService.processTranscription() - Screen analysis:', analysis);
if (analysis) {
this.events.push({
type: analysis.type,
@ -157,34 +169,40 @@ class RecorderService {
}
}
async stopRecording() {
console.log('RecorderService.stopRecording()');
this.recording = false;
if (this.silenceTimer) {
clearTimeout(this.silenceTimer);
this.silenceTimer = null;
}
await electron_1.ipcRenderer.invoke('stop-audio-recording');
electron_1.ipcRenderer.removeListener('audio-level', this.handleAudioLevel);
electron_1.ipcRenderer.removeListener('audio-chunk', this.handleAudioChunk);
electron_1.ipcRenderer.removeListener('keyboard-event', this.keyboardHandleEvent); // Remove keyboard listener
electron_1.ipcRenderer.removeListener('keyboard-event', this.keyboardHandleEvent);
if (this.currentAudioFile && fs.existsSync(this.currentAudioFile)) {
fs.unlinkSync(this.currentAudioFile);
}
return this.generateBasicCode();
const code = this.generateBasicCode();
console.log('RecorderService.stopRecording() - Generated code:', code);
return code;
}
async requestScreenshot() {
console.log('RecorderService.requestScreenshot()');
try {
const sources = await electron_1.ipcRenderer.invoke('get-screenshot');
console.log('RecorderService.requestScreenshot() - Sources:', sources);
const screenSource = sources[0];
await this.screenshotHandleEvent(null, screenSource.thumbnail);
}
catch (error) {
console.error('Error capturing screenshot:', error);
console.error('RecorderService.requestScreenshot() error:', error);
}
}
async screenshotHandleEvent(_, screenshot) {
console.log('RecorderService.screenshotHandleEvent()', { screenshot });
this.currentScreenshot = screenshot;
}
async keyboardHandleEvent(_, event) {
console.log('RecorderService.keyboardHandleEvent()', { key: event.key });
if (!this.recording)
return;
this.events.push({
@ -195,10 +213,13 @@ class RecorderService {
});
}
async mouseHandleEvent(_, event) {
console.log('RecorderService.mouseHandleEvent()', { x: event.x, y: event.y });
if (!this.recording)
return;
const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot);
console.log('RecorderService.mouseHandleEvent() - Screen analysis:', analysis);
const element = this.findElementAtPosition(analysis, event.x, event.y);
console.log('RecorderService.mouseHandleEvent() - Found element:', element);
if (element) {
this.events.push({
type: 'click',
@ -209,16 +230,21 @@ class RecorderService {
}
}
findElementAtPosition(analysis, x, y) {
//@ts-nocheck
console.log('RecorderService.findElementAtPosition()', { x, y, analysisElementsCount: analysis.elements.length });
return analysis.elements.find((element) => {
const bounds = element.bounds;
return x >= bounds.x &&
const found = x >= bounds.x &&
x <= bounds.x + bounds.width &&
y >= bounds.y &&
y <= bounds.y + bounds.height;
if (found) {
console.log('RecorderService.findElementAtPosition() - Found matching element:', element);
}
return found;
});
}
generateBasicCode() {
console.log('RecorderService.generateBasicCode()', { eventsCount: this.events.length });
let basicCode = '10 REM BotDesktop Automation Script\n';
let lineNumber = 20;
for (const event of this.events) {
@ -228,9 +254,6 @@ class RecorderService {
case 'click':
basicCode += `${lineNumber} CLICK "${event.identifier}"\n`;
break;
case 'type':
basicCode += `${lineNumber} TYPE "${event.identifier}"\n`;
break;
case 'type':
basicCode += `${lineNumber} TYPE "${event.identifier}" "${event.value}"\n`;
break;
@ -241,6 +264,7 @@ class RecorderService {
lineNumber += 10;
}
basicCode += `${lineNumber} END\n`;
console.log('RecorderService.generateBasicCode() - Generated code:', basicCode);
return basicCode;
}
}

7
package-lock.json generated
View file

@ -14,6 +14,7 @@
"debounce": "^2.2.0",
"dotenv": "^16.4.5",
"electron": "^28.0.0",
"electron-require": "^0.3.0",
"lodash": "^4.17.21",
"node-global-key-listener": "^0.3.0",
"node-mouse": "^0.0.2",
@ -3508,6 +3509,12 @@
"node": ">= 10.0.0"
}
},
"node_modules/electron-require": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/electron-require/-/electron-require-0.3.0.tgz",
"integrity": "sha512-/e3qgt6h2rxVD0I35KLsjbZKBYdJKRA7dyFyehdnVXqo5MVrWF0f0h9j0n5qWpAmr/ahETN33kv6985cHUwivw==",
"license": "MIT"
},
"node_modules/electron-to-chromium": {
"version": "1.5.47",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.47.tgz",

3
package.json
View file

@ -16,7 +16,8 @@
"debounce": "^2.2.0",
"dotenv": "^16.4.5",
"electron": "^28.0.0",
"lodash": "^4.17.21",
"electron-require": "^0.3.0",
"node-global-key-listener": "^0.3.0",
"node-mouse": "^0.0.2",
"openai": "^4.28.0",

28
src/main/main.ts
View file

@ -1,8 +1,9 @@
require('dotenv').config();
import { app, BrowserWindow, ipcMain } from 'electron';
require('electron-require');
import { app, BrowserWindow, desktopCapturer, ipcMain } from 'electron';
import * as path from 'path';
// In main.ts
import { systemPreferences } from 'electron';
import { systemPreferences } from 'electron';
import { RecorderService } from '../services/recorder.service';
import { PlayerService } from '../services/player.service';
@ -13,8 +14,11 @@ function createWindow() {
const mainWindow = new BrowserWindow({
width: 1200,
height: 800,
webPreferences: {
nodeIntegrationInWorker: true,
nodeIntegration: true,
nodeIntegrationInSubFrames: true,
contextIsolation: false,
preload: path.join(__dirname, '../preload/preload.js')
}
@ -46,22 +50,33 @@ ipcMain.handle('mouse-event', recorder.mouseHandleEvent.bind(recorder));
ipcMain.handle('keyboard-event', recorder.keyboardHandleEvent.bind(recorder));
ipcMain.handle('screenshot-captured', recorder.screenshotHandleEvent.bind(recorder));
// Handler to capture the entire screen
ipcMain.handle('get-screenshot', async () => {
console.log('get-screenshot called');
const sources = await desktopCapturer.getSources({ types: ['screen'] });
const screenSource = sources[0]; // Get the first screen source
const { thumbnail } = screenSource; // Thumbnail is a native image
return thumbnail.toPNG(); // Return the screenshot as PNG buffer
});
ipcMain.handle('start-recording', async () => {
console.log('start-recording called');
await recorder.startRecording();
});
ipcMain.handle('stop-recording', async () => {
console.log('stop-recording called');
return await recorder.stopRecording();
});
ipcMain.handle('execute-basic-code', async (_, code: string) => {
console.log('execute-basic-code called with:', code);
await player.executeBasicCode(code);
});
// Add microphone permission check for macOS
ipcMain.handle('check-microphone-permission', async () => {
console.log('check-microphone-permission called');
if (process.platform === 'darwin') {
const status = await systemPreferences.getMediaAccessStatus('microphone');
if (status !== 'granted') {
@ -70,8 +85,7 @@ ipcMain.handle('check-microphone-permission', async () => {
}
return true;
}
// On Windows/Linux, permissions are handled by the OS
return true;
return true; // On Windows/Linux, permissions are handled by the OS
});
// Enable required permissions

13
src/preload/preload.ts
View file

@ -0,0 +1,13 @@
const { ipcRenderer } = require('electron');
//@ts-nocheck
(window as any).myApi = {
sendMessage: (message: any) => {
console.log('[preload] sendMessage called with:', message);
return ipcRenderer.send('message-from-renderer', message);
},
receiveMessage: (callback: any) => {
console.log('[preload] receiveMessage registered with callback');
return ipcRenderer.on('message-from-main', (event, arg) => callback(arg));
},
};

1
src/renderer/index.html
View file

@ -3,6 +3,7 @@
<head>
<meta charset="UTF-8">
<title>BotDesktop</title>
<script>var global = global || window;</script>
<script src="https://cdn.tailwindcss.com"></script>
</head>
<body>

4
src/renderer/index.tsx
View file

@ -1,10 +1,8 @@
import React from 'react';
import ReactDOM from 'react-dom/client';
import App from '../components/App';
console.log('[renderer] Initializing React app');
ReactDOM.createRoot(
document.getElementById('root') as HTMLElement
).render(

16
src/services/player.service.ts
View file

@ -6,10 +6,12 @@ export class PlayerService {
private openAIService: OpenAIService;
constructor() {
console.log('[PlayerService] Initializing');
this.openAIService = new OpenAIService();
}
async executeBasicCode(code: string) {
console.log('[PlayerService] executeBasicCode called with:', code);
const lines = code.split('\n');
for (const line of lines) {
@ -19,29 +21,33 @@ export class PlayerService {
if (!match) continue;
const [_, command, identifier, value] = match;
console.log('[PlayerService] Executing command:', { command, identifier, value });
await this.executeCommand(command, identifier, value);
await new Promise(resolve => setTimeout(resolve, 500));
}
}
private async executeCommand(command: string, identifier: string, value?: string) {
// Capture current screen
console.log('[PlayerService] executeCommand called with:', { command, identifier, value });
const screenshotPath = await this.captureScreen();
console.log('[PlayerService] Screen captured at:', screenshotPath);
const analysis = await this.openAIService.analyzeScreen(screenshotPath);
const element = analysis.elements.find(e => e.identifier === identifier);
if (!element) throw new Error(`Element not found: ${identifier}`);
// Calculate center point of element
const centerX = element.bounds.x + element.bounds.width/2;
const centerY = element.bounds.y + element.bounds.height/2;
switch (command) {
case 'CLICK':
console.log('[PlayerService] Simulating click at:', { centerX, centerY });
await this.simulateClick(centerX, centerY);
break;
case 'TYPE':
console.log('[PlayerService] Simulating type:', { centerX, centerY, value });
await this.simulateClick(centerX, centerY);
await this.simulateTyping(value || '');
break;
@ -49,8 +55,10 @@ export class PlayerService {
}
private async captureScreen(): Promise<string> {
console.log('[PlayerService] captureScreen called');
return new Promise((resolve, reject) => {
ipcMain.once('screen-captured', (_, screenshotPath) => {
console.log('[PlayerService] Screen captured event received:', screenshotPath);
resolve(screenshotPath);
});
@ -59,8 +67,10 @@ export class PlayerService {
}
private async simulateClick(x: number, y: number): Promise<void> {
console.log('[PlayerService] simulateClick called with:', { x, y });
return new Promise((resolve) => {
ipcMain.once('click-completed', () => {
console.log('[PlayerService] Click completed');
resolve();
});
@ -69,8 +79,10 @@ export class PlayerService {
}
private async simulateTyping(text: string): Promise<void> {
console.log('[PlayerService] simulateTyping called with:', text);
return new Promise((resolve) => {
ipcMain.once('typing-completed', () => {
console.log('[PlayerService] Typing completed');
resolve();
});

70
src/services/recorder.service.ts
View file

@ -1,7 +1,6 @@
import { ipcRenderer } from 'electron';
import { AutomationEvent, ScreenAnalysis, WhisperResponse } from '../services/types';
import { OpenAIService } from '../services/openai.service';
const _ = require('lodash');
import * as path from 'path';
import * as fs from 'fs';
@ -11,13 +10,13 @@ export class RecorderService {
private openAIService: OpenAIService;
private currentScreenshot: string = '';
private lastTranscription: string = '';
private recordingProcess: any = null;
private tempDir: string;
private currentAudioFile: string = '';
private silenceTimer: NodeJS.Timeout | null = null;
private isProcessingAudio: boolean = false;
private tempDir: string;
constructor() {
console.log('RecorderService.constructor()');
this.openAIService = new OpenAIService();
this.tempDir = path.join(process.cwd(), 'temp_recordings');
if (!fs.existsSync(this.tempDir)) {
@ -26,31 +25,33 @@ export class RecorderService {
}
public async startRecording() {
console.log('RecorderService.startRecording()');
try {
this.recording = true;
this.events = [];
await this.setupAudioRecording();
await this.requestScreenshot();
ipcRenderer.on('keyboard-event', this.keyboardHandleEvent); // Listen for keyboard events
ipcRenderer.on('keyboard-event', this.keyboardHandleEvent);
} catch (error) {
console.error('Failed to start recording:', error);
console.error('RecorderService.startRecording() error:', error);
this.recording = false;
throw error;
}
}
private async setupAudioRecording() {
console.log('RecorderService.setupAudioRecording()');
try {
this.recordingProcess = await ipcRenderer.invoke('start-audio-recording');
ipcRenderer.on('audio-level', this.handleAudioLevel);
ipcRenderer.on('audio-chunk', this.handleAudioChunk);
} catch (error) {
console.error('Error setting up audio recording:', error);
console.error('RecorderService.setupAudioRecording() error:', error);
throw new Error(`Failed to setup audio recording: ${error.message}`);
}
}
private handleAudioLevel = _.debounce(async (_: any, level: number) => {
private handleAudioLevel = async (_: any, level: number) => {
console.log('RecorderService.handleAudioLevel()', { level });
if (!this.recording) return;
const SILENCE_THRESHOLD = 0.01;
@ -58,6 +59,7 @@ export class RecorderService {
if (level < SILENCE_THRESHOLD) {
if (!this.silenceTimer && !this.isProcessingAudio) {
console.log('RecorderService.handleAudioLevel() - Setting silence timer');
this.silenceTimer = setTimeout(async () => {
if (this.recording) {
await this.processSilence();
@ -66,13 +68,15 @@ export class RecorderService {
}
} else {
if (this.silenceTimer) {
console.log('RecorderService.handleAudioLevel() - Clearing silence timer');
clearTimeout(this.silenceTimer);
this.silenceTimer = null;
}
}
}, 100);
}
private handleAudioChunk = async (_: any, chunk: Buffer) => {
console.log('RecorderService.handleAudioChunk()', { chunkSize: chunk.length });
if (!this.recording) return;
try {
@ -85,34 +89,38 @@ export class RecorderService {
await this.processAudioFile(audioFilePath);
}
} catch (error) {
console.error('Error handling audio chunk:', error);
console.error('RecorderService.handleAudioChunk() error:', error);
}
};
private async processSilence() {
console.log('RecorderService.processSilence()');
if (this.isProcessingAudio) return;
this.isProcessingAudio = true;
try {
const audioFilePath = await ipcRenderer.invoke('save-audio-chunk');
console.log('RecorderService.processSilence() - Audio saved to:', audioFilePath);
if (audioFilePath) {
this.currentAudioFile = audioFilePath;
await this.processAudioFile(audioFilePath);
await this.requestScreenshot();
}
} catch (error) {
console.error('Error processing silence:', error);
console.error('RecorderService.processSilence() error:', error);
} finally {
this.isProcessingAudio = false;
}
}
private async processAudioFile(audioFilePath: string) {
console.log('RecorderService.processAudioFile()', { audioFilePath });
try {
const audioBuffer = fs.readFileSync(audioFilePath);
const transcription = await this.openAIService.transcribeAudio(
new Blob([audioBuffer], { type: 'audio/wav' })
);
console.log('RecorderService.processAudioFile() - Transcription:', transcription);
if (transcription.text.trim()) {
await this.processTranscription(transcription);
@ -120,18 +128,23 @@ export class RecorderService {
fs.unlinkSync(audioFilePath);
} catch (error) {
console.error('Error processing audio file:', error);
console.error('RecorderService.processAudioFile() error:', error);
}
}
private async processTranscription(transcription: WhisperResponse) {
console.log('RecorderService.processTranscription()', { transcription });
this.lastTranscription = transcription.text;
const cursorPosition = await ipcRenderer.invoke('get-cursor-position');
console.log('RecorderService.processTranscription() - Cursor position:', cursorPosition);
const analysis = await this.openAIService.analyzeScreenWithContext({
screenshot: this.currentScreenshot,
transcription: this.lastTranscription,
cursorPosition: await ipcRenderer.invoke('get-cursor-position')
cursorPosition
});
console.log('RecorderService.processTranscription() - Screen analysis:', analysis);
if (analysis) {
this.events.push({
@ -145,6 +158,7 @@ export class RecorderService {
}
public async stopRecording(): Promise<string> {
console.log('RecorderService.stopRecording()');
this.recording = false;
if (this.silenceTimer) {
@ -152,33 +166,38 @@ export class RecorderService {
this.silenceTimer = null;
}
await ipcRenderer.invoke('stop-audio-recording');
ipcRenderer.removeListener('audio-level', this.handleAudioLevel);
ipcRenderer.removeListener('audio-chunk', this.handleAudioChunk);
ipcRenderer.removeListener('keyboard-event', this.keyboardHandleEvent); // Remove keyboard listener
ipcRenderer.removeListener('keyboard-event', this.keyboardHandleEvent);
if (this.currentAudioFile && fs.existsSync(this.currentAudioFile)) {
fs.unlinkSync(this.currentAudioFile);
}
return this.generateBasicCode();
const code = this.generateBasicCode();
console.log('RecorderService.stopRecording() - Generated code:', code);
return code;
}
private async requestScreenshot() {
console.log('RecorderService.requestScreenshot()');
try {
const sources = await ipcRenderer.invoke('get-screenshot');
console.log('RecorderService.requestScreenshot() - Sources:', sources);
const screenSource = sources[0];
await this.screenshotHandleEvent(null, screenSource.thumbnail);
} catch (error) {
console.error('Error capturing screenshot:', error);
console.error('RecorderService.requestScreenshot() error:', error);
}
}
public async screenshotHandleEvent(_: any, screenshot: string) {
console.log('RecorderService.screenshotHandleEvent()', { screenshot });
this.currentScreenshot = screenshot;
}
public async keyboardHandleEvent(_: any, event: KeyboardEvent) {
console.log('RecorderService.keyboardHandleEvent()', { key: event.key });
if (!this.recording) return;
this.events.push({
@ -190,10 +209,14 @@ export class RecorderService {
}
public async mouseHandleEvent(_: any, event: any) {
console.log('RecorderService.mouseHandleEvent()', { x: event.x, y: event.y });
if (!this.recording) return;
const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot);
console.log('RecorderService.mouseHandleEvent() - Screen analysis:', analysis);
const element = this.findElementAtPosition(analysis, event.x, event.y);
console.log('RecorderService.mouseHandleEvent() - Found element:', element);
if (element) {
this.events.push({
@ -206,17 +229,22 @@ export class RecorderService {
}
private findElementAtPosition(analysis: ScreenAnalysis, x: number, y: number) {
//@ts-nocheck
console.log('RecorderService.findElementAtPosition()', { x, y, analysisElementsCount: analysis.elements.length });
return analysis.elements.find((element) => {
const bounds = element.bounds;
return x >= bounds.x &&
const found = x >= bounds.x &&
x <= bounds.x + bounds.width &&
y >= bounds.y &&
y <= bounds.y + bounds.height;
if (found) {
console.log('RecorderService.findElementAtPosition() - Found matching element:', element);
}
return found;
});
}
private generateBasicCode(): string {
console.log('RecorderService.generateBasicCode()', { eventsCount: this.events.length });
let basicCode = '10 REM BotDesktop Automation Script\n';
let lineNumber = 20;
@ -228,9 +256,6 @@ export class RecorderService {
case 'click':
basicCode += `${lineNumber} CLICK "${event.identifier}"\n`;
break;
case 'type':
basicCode += `${lineNumber} TYPE "${event.identifier}"\n`;
break;
case 'type':
basicCode += `${lineNumber} TYPE "${event.identifier}" "${event.value}"\n`;
break;
@ -242,6 +267,7 @@ export class RecorderService {
}
basicCode += `${lineNumber} END\n`;
console.log('RecorderService.generateBasicCode() - Generated code:', basicCode);
return basicCode;
}
}

2
src/services/types.ts
View file

@ -11,6 +11,8 @@ export interface AutomationAction {
};
}
export interface AutomationEvent {
type: 'click' | 'type' | 'move';
identifier: string;

7
webpack.config.js
View file

@ -1,5 +1,7 @@
const path = require('path');
const HtmlWebpackPlugin = require('html-webpack-plugin');
// webpack.config.js
const webpack = require('webpack');
module.exports = {
devtool: 'source-map',
@ -22,7 +24,10 @@ module.exports = {
path: path.resolve(__dirname, 'dist/renderer'),
},
plugins: [
new HtmlWebpackPlugin({
new webpack.ProvidePlugin({
global: 'global', // This will make global available in your bundled code
}),
new HtmlWebpackPlugin({
template: './src/renderer/index.html'
}),
],