From c6f0933bfda787485725df0c740392d2c8e8b1fa Mon Sep 17 00:00:00 2001 From: "me@rodrigorodriguez.com" Date: Sat, 26 Oct 2024 21:21:51 -0300 Subject: [PATCH] new(all): Initial import. --- gencode.sh => cexp.sh | 0 cimp.sh | 337 ++++++++++++++++++++++++++++++ dist/main/main.js | 20 +- dist/preload/preload.js | 12 ++ dist/renderer/index.js | 1 + dist/services/player.service.js | 15 +- dist/services/recorder.service.js | 66 ++++-- package-lock.json | 7 + package.json | 3 +- src/main/main.ts | 30 ++- src/preload/preload.ts | 13 ++ src/renderer/index.html | 1 + src/renderer/index.tsx | 4 +- src/services/player.service.ts | 18 +- src/services/recorder.service.ts | 72 +++++-- src/services/types.ts | 2 + webpack.config.js | 7 +- 17 files changed, 542 insertions(+), 66 deletions(-) rename gencode.sh => cexp.sh (100%) create mode 100755 cimp.sh diff --git a/gencode.sh b/cexp.sh similarity index 100% rename from gencode.sh rename to cexp.sh diff --git a/cimp.sh b/cimp.sh new file mode 100755 index 0000000..b8eeb49 --- /dev/null +++ b/cimp.sh @@ -0,0 +1,337 @@ +#!/bin/bash + +# Create project directories +mkdir -p ./src/preload +mkdir -p ./src/renderer +mkdir -p ./src/services + +# Create preload.ts file +cat < ./src/preload/preload.ts +// File: ./src/preload/preload.ts + +const { ipcRenderer } = require('electron'); + +//@ts-nocheck +(window as any).myApi = { + //@ts-nocheck + sendMessage: (message: any) => { + console.log('preload.sendMessage', { message }); + ipcRenderer.send('message-from-renderer', message); + }, + //@ts-nocheck + receiveMessage: (callback: any) => { + console.log('preload.receiveMessage', { callback }); + ipcRenderer.on('message-from-main', (event, arg) => callback(arg)); + }, +}; +EOL + +# Create index.tsx file +cat < ./src/renderer/index.tsx +// File: ./src/renderer/index.tsx + +import React from 'react'; +import ReactDOM from 'react-dom/client'; +import App from '../components/App'; + +ReactDOM.createRoot( + document.getElementById('root') as HTMLElement +).render( + + + +); +EOL + +# Create player.service.ts file +cat < ./src/services/player.service.ts +// File: ./src/services/player.service.ts + +import { ipcMain } from 'electron'; +import { AutomationEvent, ScreenAnalysis } from './types'; +import { OpenAIService } from './openai.service'; + +export class PlayerService { + private openAIService: OpenAIService; + + constructor() { + console.log('PlayerService.constructor', {}); + this.openAIService = new OpenAIService(); + } + + async executeBasicCode(code: string) { + console.log('PlayerService.executeBasicCode', { code }); + const lines = code.split('\\n'); + + for (const line of lines) { + if (line.trim().startsWith('REM') || line.trim() === '') continue; + + const match = line.match(/^\\d+\\s+(\\w+)\\s+"([^"]+)"(?:\\s+"([^"]+)")?/); + if (!match) continue; + + const [_, command, identifier, value] = match; + await this.executeCommand(command, identifier, value); + await new Promise(resolve => setTimeout(resolve, 500)); + } + } + + private async executeCommand(command: string, identifier: string, value?: string) { + console.log('PlayerService.executeCommand', { command, identifier, value }); + const screenshotPath = await this.captureScreen(); + + const analysis = await this.openAIService.analyzeScreen(screenshotPath); + const element = analysis.elements.find(e => e.identifier === identifier); + + if (!element) throw new Error(\`Element not found: \${identifier}\`); + + const centerX = element.bounds.x + element.bounds.width / 2; + const centerY = element.bounds.y + element.bounds.height / 2; + + switch (command) { + case 'CLICK': + await this.simulateClick(centerX, centerY); + break; + case 'TYPE': + await this.simulateClick(centerX, centerY); + await this.simulateTyping(value || ''); + break; + } + } + + private async captureScreen(): Promise { + console.log('PlayerService.captureScreen', {}); + return new Promise((resolve, reject) => { + ipcMain.once('screen-captured', (_, screenshotPath) => { + resolve(screenshotPath); + }); + + ipcMain.emit('capture-screen'); + }); + } + + private async simulateClick(x: number, y: number): Promise { + console.log('PlayerService.simulateClick', { x, y }); + return new Promise((resolve) => { + ipcMain.once('click-completed', () => { + resolve(); + }); + + ipcMain.emit('simulate-click', { x, y }); + }); + } + + private async simulateTyping(text: string): Promise { + console.log('PlayerService.simulateTyping', { text }); + return new Promise((resolve) => { + ipcMain.once('typing-completed', () => { + resolve(); + }); + + ipcMain.emit('simulate-typing', { text }); + }); + } +} +EOL + +# Create types.ts file +cat < ./src/services/types.ts +// File: ./src/services/types.ts + +export interface AutomationAction { + type: 'click' | 'type' | 'move'; + identifier: string; + value?: string; + confidence: number; + bounds: { + x: number; + y: number; + width: number; + height: number; + }; +} + +export interface AutomationEvent { + type: 'click' | 'type' | 'move'; + identifier: string; + value?: string; + timestamp: number; + narration: string; +} + +export interface WhisperResponse { + text: string; + segments: any; +} + +export interface ScreenContext { + screenshot: string; + transcription: string; + cursorPosition: { x: number, y: number }; +} + +export interface ScreenAnalysis { + timestamp: number, + elements: { + identifier: string; + type: string; + bounds: { x: number; y: number; width: number; height: number }; + value?: string; + }[]; +} +EOL + +# Create recorder.service.ts file +cat < ./src/services/recorder.service.ts +// File: ./src/services/recorder.service.ts + +const { ipcRenderer } = require('electron'); // Require ipcRender +import { AutomationEvent, ScreenAnalysis, WhisperResponse } from '../services/types'; +import { OpenAIService } from '../services/openai.service'; +import * as path from 'path'; +import * as fs from 'fs'; + +export class RecorderService { + private events: AutomationEvent[] = []; + private recording: boolean = false; + private openAIService: OpenAIService; + private currentScreenshot: string = ''; + private lastTranscription: string = ''; + private recordingProcess: any = null; + private tempDir: string; + private currentAudioFile: string = ''; + private silenceTimer: NodeJS.Timeout | null = null; + private isProcessingAudio: boolean = false; + + constructor() { + console.log('RecorderService.constructor', {}); + this.openAIService = new OpenAIService(); + this.tempDir = path.join(process.cwd(), 'temp_recordings'); + if (!fs.existsSync(this.tempDir)) { + fs.mkdirSync(this.tempDir, { recursive: true }); + } + } + + public async startRecording() { + console.log('RecorderService.startRecording', {}); + try { + this.recording = true; + this.events = []; + await this.setupAudioRecording(); + await this.requestScreenshot(); + ipcRenderer.on('keyboard-event', this.keyboardHandleEvent); + } catch (error) { + console.error('Failed to start recording:', error); + this.recording = false; + throw error; + } + } + + private async setupAudioRecording() { + console.log('RecorderService.setupAudioRecording', {}); + try { + ipcRenderer.on('audio-level', this.handleAudioLevel); + ipcRenderer.on('audio-chunk', this.handleAudioChunk); + } catch (error) { + console.error('Error setting up audio recording:', error); + throw new Error(\`Failed to setup audio recording: \${error.message}\`); + } + } + + private handleAudioLevel = async (_: any, level: number) => { + console.log('RecorderService.handleAudioLevel', { level }); + if (!this.recording) return; + + const SILENCE_THRESHOLD = 0.01; + const SILENCE_DURATION = 1000; + + if (level < SILENCE_THRESHOLD) { + if (!this.silenceTimer && !this.isProcessingAudio) { + this.silenceTimer = setTimeout(async () => { + if (this.recording) { + await this.processSilence(); + } + }, SILENCE_DURATION); + } + } else { + if (this.silenceTimer) { + clearTimeout(this.silenceTimer); + this.silenceTimer = null; + } + } + } + + private handleAudioChunk = async (_: any, chunk: Buffer) => { + console.log('RecorderService.handleAudioChunk', { chunk }); + if (!this.recording) return; + + try { + const audioFilePath = path.join(this.tempDir, \`audio-\${Date.now()}.wav\`); + fs.writeFileSync(audioFilePath, chunk); + + if (this.silenceTimer) { + clearTimeout(this.silenceTimer); + this.silenceTimer = null; + await this.processAudioFile(audioFilePath); + } + } catch (error) { + console.error('Error handling audio chunk:', error); + } + }; + + private async processSilence() { + console.log('RecorderService.processSilence', {}); + if (this.isProcessingAudio) return; + + this.isProcessingAudio = true; + try { + const audioFilePath = await ipcRenderer.invoke('save-audio-chunk'); + if (audioFilePath) { + this.currentAudioFile = audioFilePath; + await this.processAudioFile(audioFilePath); + await this.requestScreenshot(); + } + } catch (error) { + console.error('Error processing silence:', error); + } finally { + this.isProcessingAudio = false; + } + } + + private async processAudioFile(audioFilePath: string) { + console.log('RecorderService.processAudioFile', { audioFilePath }); + const transcription = await this.openAIService.transcribeAudio(audioFilePath); + this.lastTranscription = transcription; + await this.requestScreenshot(); + } + + private async requestScreenshot() { + console.log('RecorderService.requestScreenshot', {}); + await ipcRenderer.invoke('request-screenshot'); + } + + private keyboardHandleEvent = async (_: any, event: any) => { + console.log('RecorderService.keyboardHandleEvent', { event }); + if (!this.recording) return; + + const automationEvent: AutomationEvent = { + type: 'keyboard', + identifier: event.key, + timestamp: Date.now(), + narration: this.lastTranscription, + }; + + this.events.push(automationEvent); + }; + + public async stopRecording() { + console.log('RecorderService.stopRecording', {}); + try { + this.recording = false; + ipcRenderer.removeListener('keyboard-event', this.keyboardHandleEvent); + await ipcRenderer.invoke('stop-audio-recording'); + } catch (error) { + console.error('Failed to stop recording:', error); + } + } +} +EOL diff --git a/dist/main/main.js b/dist/main/main.js index f9c8d00..517e3d2 100644 --- a/dist/main/main.js +++ b/dist/main/main.js @@ -24,9 +24,9 @@ var __importStar = (this && this.__importStar) || function (mod) { }; Object.defineProperty(exports, "__esModule", { value: true }); require('dotenv').config(); +require('electron-require'); const electron_1 = require("electron"); const path = __importStar(require("path")); -// In main.ts const electron_2 = require("electron"); const recorder_service_1 = require("../services/recorder.service"); const player_service_1 = require("../services/player.service"); @@ -37,7 +37,9 @@ function createWindow() { width: 1200, height: 800, webPreferences: { + nodeIntegrationInWorker: true, nodeIntegration: true, + nodeIntegrationInSubFrames: true, contextIsolation: false, preload: path.join(__dirname, '../preload/preload.js') } @@ -64,17 +66,28 @@ electron_1.app.on('activate', () => { electron_1.ipcMain.handle('mouse-event', recorder.mouseHandleEvent.bind(recorder)); electron_1.ipcMain.handle('keyboard-event', recorder.keyboardHandleEvent.bind(recorder)); electron_1.ipcMain.handle('screenshot-captured', recorder.screenshotHandleEvent.bind(recorder)); +// Handler to capture the entire screen +electron_1.ipcMain.handle('get-screenshot', async () => { + console.log('get-screenshot called'); + const sources = await electron_1.desktopCapturer.getSources({ types: ['screen'] }); + const screenSource = sources[0]; // Get the first screen source + const { thumbnail } = screenSource; // Thumbnail is a native image + return thumbnail.toPNG(); // Return the screenshot as PNG buffer +}); electron_1.ipcMain.handle('start-recording', async () => { + console.log('start-recording called'); await recorder.startRecording(); }); electron_1.ipcMain.handle('stop-recording', async () => { + console.log('stop-recording called'); return await recorder.stopRecording(); }); electron_1.ipcMain.handle('execute-basic-code', async (_, code) => { + console.log('execute-basic-code called with:', code); await player.executeBasicCode(code); }); -// Add microphone permission check for macOS electron_1.ipcMain.handle('check-microphone-permission', async () => { + console.log('check-microphone-permission called'); if (process.platform === 'darwin') { const status = await electron_2.systemPreferences.getMediaAccessStatus('microphone'); if (status !== 'granted') { @@ -83,8 +96,7 @@ electron_1.ipcMain.handle('check-microphone-permission', async () => { } return true; } - // On Windows/Linux, permissions are handled by the OS - return true; + return true; // On Windows/Linux, permissions are handled by the OS }); // Enable required permissions electron_1.app.commandLine.appendSwitch('enable-speech-dispatcher'); diff --git a/dist/preload/preload.js b/dist/preload/preload.js index e69de29..158bcee 100644 --- a/dist/preload/preload.js +++ b/dist/preload/preload.js @@ -0,0 +1,12 @@ +const { ipcRenderer } = require('electron'); +//@ts-nocheck +window.myApi = { + sendMessage: (message) => { + console.log('[preload] sendMessage called with:', message); + return ipcRenderer.send('message-from-renderer', message); + }, + receiveMessage: (callback) => { + console.log('[preload] receiveMessage registered with callback'); + return ipcRenderer.on('message-from-main', (event, arg) => callback(arg)); + }, +}; diff --git a/dist/renderer/index.js b/dist/renderer/index.js index 7872e63..0a53067 100644 --- a/dist/renderer/index.js +++ b/dist/renderer/index.js @@ -6,5 +6,6 @@ Object.defineProperty(exports, "__esModule", { value: true }); const react_1 = __importDefault(require("react")); const client_1 = __importDefault(require("react-dom/client")); const App_1 = __importDefault(require("../components/App")); +console.log('[renderer] Initializing React app'); client_1.default.createRoot(document.getElementById('root')).render(react_1.default.createElement(react_1.default.StrictMode, null, react_1.default.createElement(App_1.default, null))); diff --git a/dist/services/player.service.js b/dist/services/player.service.js index 1b7512b..a795a94 100644 --- a/dist/services/player.service.js +++ b/dist/services/player.service.js @@ -5,9 +5,11 @@ const electron_1 = require("electron"); const openai_service_1 = require("./openai.service"); class PlayerService { constructor() { + console.log('[PlayerService] Initializing'); this.openAIService = new openai_service_1.OpenAIService(); } async executeBasicCode(code) { + console.log('[PlayerService] executeBasicCode called with:', code); const lines = code.split('\n'); for (const line of lines) { if (line.trim().startsWith('REM') || line.trim() === '') @@ -16,49 +18,58 @@ class PlayerService { if (!match) continue; const [_, command, identifier, value] = match; + console.log('[PlayerService] Executing command:', { command, identifier, value }); await this.executeCommand(command, identifier, value); await new Promise(resolve => setTimeout(resolve, 500)); } } async executeCommand(command, identifier, value) { - // Capture current screen + console.log('[PlayerService] executeCommand called with:', { command, identifier, value }); const screenshotPath = await this.captureScreen(); + console.log('[PlayerService] Screen captured at:', screenshotPath); const analysis = await this.openAIService.analyzeScreen(screenshotPath); const element = analysis.elements.find(e => e.identifier === identifier); if (!element) throw new Error(`Element not found: ${identifier}`); - // Calculate center point of element const centerX = element.bounds.x + element.bounds.width / 2; const centerY = element.bounds.y + element.bounds.height / 2; switch (command) { case 'CLICK': + console.log('[PlayerService] Simulating click at:', { centerX, centerY }); await this.simulateClick(centerX, centerY); break; case 'TYPE': + console.log('[PlayerService] Simulating type:', { centerX, centerY, value }); await this.simulateClick(centerX, centerY); await this.simulateTyping(value || ''); break; } } async captureScreen() { + console.log('[PlayerService] captureScreen called'); return new Promise((resolve, reject) => { electron_1.ipcMain.once('screen-captured', (_, screenshotPath) => { + console.log('[PlayerService] Screen captured event received:', screenshotPath); resolve(screenshotPath); }); electron_1.ipcMain.emit('capture-screen'); }); } async simulateClick(x, y) { + console.log('[PlayerService] simulateClick called with:', { x, y }); return new Promise((resolve) => { electron_1.ipcMain.once('click-completed', () => { + console.log('[PlayerService] Click completed'); resolve(); }); electron_1.ipcMain.emit('simulate-click', { x, y }); }); } async simulateTyping(text) { + console.log('[PlayerService] simulateTyping called with:', text); return new Promise((resolve) => { electron_1.ipcMain.once('typing-completed', () => { + console.log('[PlayerService] Typing completed'); resolve(); }); electron_1.ipcMain.emit('simulate-typing', { text }); diff --git a/dist/services/recorder.service.js b/dist/services/recorder.service.js index cc24026..0ac1c5c 100644 --- a/dist/services/recorder.service.js +++ b/dist/services/recorder.service.js @@ -26,7 +26,6 @@ Object.defineProperty(exports, "__esModule", { value: true }); exports.RecorderService = void 0; const electron_1 = require("electron"); const openai_service_1 = require("../services/openai.service"); -const _ = require('lodash'); const path = __importStar(require("path")); const fs = __importStar(require("fs")); class RecorderService { @@ -35,17 +34,18 @@ class RecorderService { this.recording = false; this.currentScreenshot = ''; this.lastTranscription = ''; - this.recordingProcess = null; this.currentAudioFile = ''; this.silenceTimer = null; this.isProcessingAudio = false; - this.handleAudioLevel = _.debounce(async (_, level) => { + this.handleAudioLevel = async (_, level) => { + console.log('RecorderService.handleAudioLevel()', { level }); if (!this.recording) return; const SILENCE_THRESHOLD = 0.01; const SILENCE_DURATION = 1000; if (level < SILENCE_THRESHOLD) { if (!this.silenceTimer && !this.isProcessingAudio) { + console.log('RecorderService.handleAudioLevel() - Setting silence timer'); this.silenceTimer = setTimeout(async () => { if (this.recording) { await this.processSilence(); @@ -55,12 +55,14 @@ class RecorderService { } else { if (this.silenceTimer) { + console.log('RecorderService.handleAudioLevel() - Clearing silence timer'); clearTimeout(this.silenceTimer); this.silenceTimer = null; } } - }, 100); + }; this.handleAudioChunk = async (_, chunk) => { + console.log('RecorderService.handleAudioChunk()', { chunkSize: chunk.length }); if (!this.recording) return; try { @@ -73,9 +75,10 @@ class RecorderService { } } catch (error) { - console.error('Error handling audio chunk:', error); + console.error('RecorderService.handleAudioChunk() error:', error); } }; + console.log('RecorderService.constructor()'); this.openAIService = new openai_service_1.OpenAIService(); this.tempDir = path.join(process.cwd(), 'temp_recordings'); if (!fs.existsSync(this.tempDir)) { @@ -83,36 +86,39 @@ class RecorderService { } } async startRecording() { + console.log('RecorderService.startRecording()'); try { this.recording = true; this.events = []; await this.setupAudioRecording(); await this.requestScreenshot(); - electron_1.ipcRenderer.on('keyboard-event', this.keyboardHandleEvent); // Listen for keyboard events + electron_1.ipcRenderer.on('keyboard-event', this.keyboardHandleEvent); } catch (error) { - console.error('Failed to start recording:', error); + console.error('RecorderService.startRecording() error:', error); this.recording = false; throw error; } } async setupAudioRecording() { + console.log('RecorderService.setupAudioRecording()'); try { - this.recordingProcess = await electron_1.ipcRenderer.invoke('start-audio-recording'); electron_1.ipcRenderer.on('audio-level', this.handleAudioLevel); electron_1.ipcRenderer.on('audio-chunk', this.handleAudioChunk); } catch (error) { - console.error('Error setting up audio recording:', error); + console.error('RecorderService.setupAudioRecording() error:', error); throw new Error(`Failed to setup audio recording: ${error.message}`); } } async processSilence() { + console.log('RecorderService.processSilence()'); if (this.isProcessingAudio) return; this.isProcessingAudio = true; try { const audioFilePath = await electron_1.ipcRenderer.invoke('save-audio-chunk'); + console.log('RecorderService.processSilence() - Audio saved to:', audioFilePath); if (audioFilePath) { this.currentAudioFile = audioFilePath; await this.processAudioFile(audioFilePath); @@ -120,32 +126,38 @@ class RecorderService { } } catch (error) { - console.error('Error processing silence:', error); + console.error('RecorderService.processSilence() error:', error); } finally { this.isProcessingAudio = false; } } async processAudioFile(audioFilePath) { + console.log('RecorderService.processAudioFile()', { audioFilePath }); try { const audioBuffer = fs.readFileSync(audioFilePath); const transcription = await this.openAIService.transcribeAudio(new Blob([audioBuffer], { type: 'audio/wav' })); + console.log('RecorderService.processAudioFile() - Transcription:', transcription); if (transcription.text.trim()) { await this.processTranscription(transcription); } fs.unlinkSync(audioFilePath); } catch (error) { - console.error('Error processing audio file:', error); + console.error('RecorderService.processAudioFile() error:', error); } } async processTranscription(transcription) { + console.log('RecorderService.processTranscription()', { transcription }); this.lastTranscription = transcription.text; + const cursorPosition = await electron_1.ipcRenderer.invoke('get-cursor-position'); + console.log('RecorderService.processTranscription() - Cursor position:', cursorPosition); const analysis = await this.openAIService.analyzeScreenWithContext({ screenshot: this.currentScreenshot, transcription: this.lastTranscription, - cursorPosition: await electron_1.ipcRenderer.invoke('get-cursor-position') + cursorPosition }); + console.log('RecorderService.processTranscription() - Screen analysis:', analysis); if (analysis) { this.events.push({ type: analysis.type, @@ -157,34 +169,40 @@ class RecorderService { } } async stopRecording() { + console.log('RecorderService.stopRecording()'); this.recording = false; if (this.silenceTimer) { clearTimeout(this.silenceTimer); this.silenceTimer = null; } - await electron_1.ipcRenderer.invoke('stop-audio-recording'); electron_1.ipcRenderer.removeListener('audio-level', this.handleAudioLevel); electron_1.ipcRenderer.removeListener('audio-chunk', this.handleAudioChunk); - electron_1.ipcRenderer.removeListener('keyboard-event', this.keyboardHandleEvent); // Remove keyboard listener + electron_1.ipcRenderer.removeListener('keyboard-event', this.keyboardHandleEvent); if (this.currentAudioFile && fs.existsSync(this.currentAudioFile)) { fs.unlinkSync(this.currentAudioFile); } - return this.generateBasicCode(); + const code = this.generateBasicCode(); + console.log('RecorderService.stopRecording() - Generated code:', code); + return code; } async requestScreenshot() { + console.log('RecorderService.requestScreenshot()'); try { const sources = await electron_1.ipcRenderer.invoke('get-screenshot'); + console.log('RecorderService.requestScreenshot() - Sources:', sources); const screenSource = sources[0]; await this.screenshotHandleEvent(null, screenSource.thumbnail); } catch (error) { - console.error('Error capturing screenshot:', error); + console.error('RecorderService.requestScreenshot() error:', error); } } async screenshotHandleEvent(_, screenshot) { + console.log('RecorderService.screenshotHandleEvent()', { screenshot }); this.currentScreenshot = screenshot; } async keyboardHandleEvent(_, event) { + console.log('RecorderService.keyboardHandleEvent()', { key: event.key }); if (!this.recording) return; this.events.push({ @@ -195,10 +213,13 @@ class RecorderService { }); } async mouseHandleEvent(_, event) { + console.log('RecorderService.mouseHandleEvent()', { x: event.x, y: event.y }); if (!this.recording) return; const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot); + console.log('RecorderService.mouseHandleEvent() - Screen analysis:', analysis); const element = this.findElementAtPosition(analysis, event.x, event.y); + console.log('RecorderService.mouseHandleEvent() - Found element:', element); if (element) { this.events.push({ type: 'click', @@ -209,16 +230,21 @@ class RecorderService { } } findElementAtPosition(analysis, x, y) { - //@ts-nocheck + console.log('RecorderService.findElementAtPosition()', { x, y, analysisElementsCount: analysis.elements.length }); return analysis.elements.find((element) => { const bounds = element.bounds; - return x >= bounds.x && + const found = x >= bounds.x && x <= bounds.x + bounds.width && y >= bounds.y && y <= bounds.y + bounds.height; + if (found) { + console.log('RecorderService.findElementAtPosition() - Found matching element:', element); + } + return found; }); } generateBasicCode() { + console.log('RecorderService.generateBasicCode()', { eventsCount: this.events.length }); let basicCode = '10 REM BotDesktop Automation Script\n'; let lineNumber = 20; for (const event of this.events) { @@ -228,9 +254,6 @@ class RecorderService { case 'click': basicCode += `${lineNumber} CLICK "${event.identifier}"\n`; break; - case 'type': - basicCode += `${lineNumber} TYPE "${event.identifier}"\n`; - break; case 'type': basicCode += `${lineNumber} TYPE "${event.identifier}" "${event.value}"\n`; break; @@ -241,6 +264,7 @@ class RecorderService { lineNumber += 10; } basicCode += `${lineNumber} END\n`; + console.log('RecorderService.generateBasicCode() - Generated code:', basicCode); return basicCode; } } diff --git a/package-lock.json b/package-lock.json index 9384bb4..275b7f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "debounce": "^2.2.0", "dotenv": "^16.4.5", "electron": "^28.0.0", + "electron-require": "^0.3.0", "lodash": "^4.17.21", "node-global-key-listener": "^0.3.0", "node-mouse": "^0.0.2", @@ -3508,6 +3509,12 @@ "node": ">= 10.0.0" } }, + "node_modules/electron-require": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/electron-require/-/electron-require-0.3.0.tgz", + "integrity": "sha512-/e3qgt6h2rxVD0I35KLsjbZKBYdJKRA7dyFyehdnVXqo5MVrWF0f0h9j0n5qWpAmr/ahETN33kv6985cHUwivw==", + "license": "MIT" + }, "node_modules/electron-to-chromium": { "version": "1.5.47", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.47.tgz", diff --git a/package.json b/package.json index 6f222f4..325056e 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "debounce": "^2.2.0", "dotenv": "^16.4.5", "electron": "^28.0.0", - "lodash": "^4.17.21", + "electron-require": "^0.3.0", + "node-global-key-listener": "^0.3.0", "node-mouse": "^0.0.2", "openai": "^4.28.0", diff --git a/src/main/main.ts b/src/main/main.ts index fc29b63..6ac3348 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -1,8 +1,9 @@ require('dotenv').config(); -import { app, BrowserWindow, ipcMain } from 'electron'; +require('electron-require'); + +import { app, BrowserWindow, desktopCapturer, ipcMain } from 'electron'; import * as path from 'path'; -// In main.ts -import { systemPreferences } from 'electron'; +import { systemPreferences } from 'electron'; import { RecorderService } from '../services/recorder.service'; import { PlayerService } from '../services/player.service'; @@ -13,8 +14,11 @@ function createWindow() { const mainWindow = new BrowserWindow({ width: 1200, height: 800, + webPreferences: { + nodeIntegrationInWorker: true, nodeIntegration: true, + nodeIntegrationInSubFrames: true, contextIsolation: false, preload: path.join(__dirname, '../preload/preload.js') } @@ -46,22 +50,33 @@ ipcMain.handle('mouse-event', recorder.mouseHandleEvent.bind(recorder)); ipcMain.handle('keyboard-event', recorder.keyboardHandleEvent.bind(recorder)); ipcMain.handle('screenshot-captured', recorder.screenshotHandleEvent.bind(recorder)); +// Handler to capture the entire screen +ipcMain.handle('get-screenshot', async () => { + console.log('get-screenshot called'); + const sources = await desktopCapturer.getSources({ types: ['screen'] }); + const screenSource = sources[0]; // Get the first screen source + + const { thumbnail } = screenSource; // Thumbnail is a native image + return thumbnail.toPNG(); // Return the screenshot as PNG buffer +}); ipcMain.handle('start-recording', async () => { + console.log('start-recording called'); await recorder.startRecording(); }); ipcMain.handle('stop-recording', async () => { + console.log('stop-recording called'); return await recorder.stopRecording(); }); ipcMain.handle('execute-basic-code', async (_, code: string) => { + console.log('execute-basic-code called with:', code); await player.executeBasicCode(code); }); - -// Add microphone permission check for macOS ipcMain.handle('check-microphone-permission', async () => { + console.log('check-microphone-permission called'); if (process.platform === 'darwin') { const status = await systemPreferences.getMediaAccessStatus('microphone'); if (status !== 'granted') { @@ -70,9 +85,8 @@ ipcMain.handle('check-microphone-permission', async () => { } return true; } - // On Windows/Linux, permissions are handled by the OS - return true; + return true; // On Windows/Linux, permissions are handled by the OS }); // Enable required permissions -app.commandLine.appendSwitch('enable-speech-dispatcher'); \ No newline at end of file +app.commandLine.appendSwitch('enable-speech-dispatcher'); diff --git a/src/preload/preload.ts b/src/preload/preload.ts index e69de29..ffed7db 100644 --- a/src/preload/preload.ts +++ b/src/preload/preload.ts @@ -0,0 +1,13 @@ +const { ipcRenderer } = require('electron'); + +//@ts-nocheck +(window as any).myApi = { + sendMessage: (message: any) => { + console.log('[preload] sendMessage called with:', message); + return ipcRenderer.send('message-from-renderer', message); + }, + receiveMessage: (callback: any) => { + console.log('[preload] receiveMessage registered with callback'); + return ipcRenderer.on('message-from-main', (event, arg) => callback(arg)); + }, +}; diff --git a/src/renderer/index.html b/src/renderer/index.html index 2001fb6..ef787dc 100644 --- a/src/renderer/index.html +++ b/src/renderer/index.html @@ -3,6 +3,7 @@ BotDesktop + diff --git a/src/renderer/index.tsx b/src/renderer/index.tsx index b822859..758f12e 100644 --- a/src/renderer/index.tsx +++ b/src/renderer/index.tsx @@ -1,10 +1,8 @@ import React from 'react'; import ReactDOM from 'react-dom/client'; - - - import App from '../components/App'; +console.log('[renderer] Initializing React app'); ReactDOM.createRoot( document.getElementById('root') as HTMLElement ).render( diff --git a/src/services/player.service.ts b/src/services/player.service.ts index ad56b7e..98074d5 100644 --- a/src/services/player.service.ts +++ b/src/services/player.service.ts @@ -6,10 +6,12 @@ export class PlayerService { private openAIService: OpenAIService; constructor() { + console.log('[PlayerService] Initializing'); this.openAIService = new OpenAIService(); } async executeBasicCode(code: string) { + console.log('[PlayerService] executeBasicCode called with:', code); const lines = code.split('\n'); for (const line of lines) { @@ -19,29 +21,33 @@ export class PlayerService { if (!match) continue; const [_, command, identifier, value] = match; + console.log('[PlayerService] Executing command:', { command, identifier, value }); await this.executeCommand(command, identifier, value); await new Promise(resolve => setTimeout(resolve, 500)); } } private async executeCommand(command: string, identifier: string, value?: string) { - // Capture current screen + console.log('[PlayerService] executeCommand called with:', { command, identifier, value }); + const screenshotPath = await this.captureScreen(); + console.log('[PlayerService] Screen captured at:', screenshotPath); const analysis = await this.openAIService.analyzeScreen(screenshotPath); const element = analysis.elements.find(e => e.identifier === identifier); if (!element) throw new Error(`Element not found: ${identifier}`); - // Calculate center point of element const centerX = element.bounds.x + element.bounds.width/2; const centerY = element.bounds.y + element.bounds.height/2; switch (command) { case 'CLICK': + console.log('[PlayerService] Simulating click at:', { centerX, centerY }); await this.simulateClick(centerX, centerY); break; case 'TYPE': + console.log('[PlayerService] Simulating type:', { centerX, centerY, value }); await this.simulateClick(centerX, centerY); await this.simulateTyping(value || ''); break; @@ -49,8 +55,10 @@ export class PlayerService { } private async captureScreen(): Promise { + console.log('[PlayerService] captureScreen called'); return new Promise((resolve, reject) => { ipcMain.once('screen-captured', (_, screenshotPath) => { + console.log('[PlayerService] Screen captured event received:', screenshotPath); resolve(screenshotPath); }); @@ -59,8 +67,10 @@ export class PlayerService { } private async simulateClick(x: number, y: number): Promise { + console.log('[PlayerService] simulateClick called with:', { x, y }); return new Promise((resolve) => { ipcMain.once('click-completed', () => { + console.log('[PlayerService] Click completed'); resolve(); }); @@ -69,12 +79,14 @@ export class PlayerService { } private async simulateTyping(text: string): Promise { + console.log('[PlayerService] simulateTyping called with:', text); return new Promise((resolve) => { ipcMain.once('typing-completed', () => { + console.log('[PlayerService] Typing completed'); resolve(); }); ipcMain.emit('simulate-typing', { text }); }); } -} \ No newline at end of file +} diff --git a/src/services/recorder.service.ts b/src/services/recorder.service.ts index ba7998a..46e45e2 100644 --- a/src/services/recorder.service.ts +++ b/src/services/recorder.service.ts @@ -1,7 +1,6 @@ import { ipcRenderer } from 'electron'; import { AutomationEvent, ScreenAnalysis, WhisperResponse } from '../services/types'; import { OpenAIService } from '../services/openai.service'; -const _ = require('lodash'); import * as path from 'path'; import * as fs from 'fs'; @@ -11,13 +10,13 @@ export class RecorderService { private openAIService: OpenAIService; private currentScreenshot: string = ''; private lastTranscription: string = ''; - private recordingProcess: any = null; - private tempDir: string; private currentAudioFile: string = ''; private silenceTimer: NodeJS.Timeout | null = null; private isProcessingAudio: boolean = false; + private tempDir: string; constructor() { + console.log('RecorderService.constructor()'); this.openAIService = new OpenAIService(); this.tempDir = path.join(process.cwd(), 'temp_recordings'); if (!fs.existsSync(this.tempDir)) { @@ -26,31 +25,33 @@ export class RecorderService { } public async startRecording() { + console.log('RecorderService.startRecording()'); try { this.recording = true; this.events = []; await this.setupAudioRecording(); await this.requestScreenshot(); - ipcRenderer.on('keyboard-event', this.keyboardHandleEvent); // Listen for keyboard events + ipcRenderer.on('keyboard-event', this.keyboardHandleEvent); } catch (error) { - console.error('Failed to start recording:', error); + console.error('RecorderService.startRecording() error:', error); this.recording = false; throw error; } } private async setupAudioRecording() { + console.log('RecorderService.setupAudioRecording()'); try { - this.recordingProcess = await ipcRenderer.invoke('start-audio-recording'); ipcRenderer.on('audio-level', this.handleAudioLevel); ipcRenderer.on('audio-chunk', this.handleAudioChunk); } catch (error) { - console.error('Error setting up audio recording:', error); + console.error('RecorderService.setupAudioRecording() error:', error); throw new Error(`Failed to setup audio recording: ${error.message}`); } } - private handleAudioLevel = _.debounce(async (_: any, level: number) => { + private handleAudioLevel = async (_: any, level: number) => { + console.log('RecorderService.handleAudioLevel()', { level }); if (!this.recording) return; const SILENCE_THRESHOLD = 0.01; @@ -58,6 +59,7 @@ export class RecorderService { if (level < SILENCE_THRESHOLD) { if (!this.silenceTimer && !this.isProcessingAudio) { + console.log('RecorderService.handleAudioLevel() - Setting silence timer'); this.silenceTimer = setTimeout(async () => { if (this.recording) { await this.processSilence(); @@ -66,13 +68,15 @@ export class RecorderService { } } else { if (this.silenceTimer) { + console.log('RecorderService.handleAudioLevel() - Clearing silence timer'); clearTimeout(this.silenceTimer); this.silenceTimer = null; } } - }, 100); + } private handleAudioChunk = async (_: any, chunk: Buffer) => { + console.log('RecorderService.handleAudioChunk()', { chunkSize: chunk.length }); if (!this.recording) return; try { @@ -85,34 +89,38 @@ export class RecorderService { await this.processAudioFile(audioFilePath); } } catch (error) { - console.error('Error handling audio chunk:', error); + console.error('RecorderService.handleAudioChunk() error:', error); } }; private async processSilence() { + console.log('RecorderService.processSilence()'); if (this.isProcessingAudio) return; this.isProcessingAudio = true; try { const audioFilePath = await ipcRenderer.invoke('save-audio-chunk'); + console.log('RecorderService.processSilence() - Audio saved to:', audioFilePath); if (audioFilePath) { this.currentAudioFile = audioFilePath; await this.processAudioFile(audioFilePath); await this.requestScreenshot(); } } catch (error) { - console.error('Error processing silence:', error); + console.error('RecorderService.processSilence() error:', error); } finally { this.isProcessingAudio = false; } } private async processAudioFile(audioFilePath: string) { + console.log('RecorderService.processAudioFile()', { audioFilePath }); try { const audioBuffer = fs.readFileSync(audioFilePath); const transcription = await this.openAIService.transcribeAudio( new Blob([audioBuffer], { type: 'audio/wav' }) ); + console.log('RecorderService.processAudioFile() - Transcription:', transcription); if (transcription.text.trim()) { await this.processTranscription(transcription); @@ -120,18 +128,23 @@ export class RecorderService { fs.unlinkSync(audioFilePath); } catch (error) { - console.error('Error processing audio file:', error); + console.error('RecorderService.processAudioFile() error:', error); } } private async processTranscription(transcription: WhisperResponse) { + console.log('RecorderService.processTranscription()', { transcription }); this.lastTranscription = transcription.text; + const cursorPosition = await ipcRenderer.invoke('get-cursor-position'); + console.log('RecorderService.processTranscription() - Cursor position:', cursorPosition); + const analysis = await this.openAIService.analyzeScreenWithContext({ screenshot: this.currentScreenshot, transcription: this.lastTranscription, - cursorPosition: await ipcRenderer.invoke('get-cursor-position') + cursorPosition }); + console.log('RecorderService.processTranscription() - Screen analysis:', analysis); if (analysis) { this.events.push({ @@ -145,6 +158,7 @@ export class RecorderService { } public async stopRecording(): Promise { + console.log('RecorderService.stopRecording()'); this.recording = false; if (this.silenceTimer) { @@ -152,33 +166,38 @@ export class RecorderService { this.silenceTimer = null; } - await ipcRenderer.invoke('stop-audio-recording'); ipcRenderer.removeListener('audio-level', this.handleAudioLevel); ipcRenderer.removeListener('audio-chunk', this.handleAudioChunk); - ipcRenderer.removeListener('keyboard-event', this.keyboardHandleEvent); // Remove keyboard listener + ipcRenderer.removeListener('keyboard-event', this.keyboardHandleEvent); if (this.currentAudioFile && fs.existsSync(this.currentAudioFile)) { fs.unlinkSync(this.currentAudioFile); } - return this.generateBasicCode(); + const code = this.generateBasicCode(); + console.log('RecorderService.stopRecording() - Generated code:', code); + return code; } private async requestScreenshot() { + console.log('RecorderService.requestScreenshot()'); try { const sources = await ipcRenderer.invoke('get-screenshot'); + console.log('RecorderService.requestScreenshot() - Sources:', sources); const screenSource = sources[0]; await this.screenshotHandleEvent(null, screenSource.thumbnail); } catch (error) { - console.error('Error capturing screenshot:', error); + console.error('RecorderService.requestScreenshot() error:', error); } } public async screenshotHandleEvent(_: any, screenshot: string) { + console.log('RecorderService.screenshotHandleEvent()', { screenshot }); this.currentScreenshot = screenshot; } public async keyboardHandleEvent(_: any, event: KeyboardEvent) { + console.log('RecorderService.keyboardHandleEvent()', { key: event.key }); if (!this.recording) return; this.events.push({ @@ -190,10 +209,14 @@ export class RecorderService { } public async mouseHandleEvent(_: any, event: any) { + console.log('RecorderService.mouseHandleEvent()', { x: event.x, y: event.y }); if (!this.recording) return; const analysis = await this.openAIService.analyzeScreen(this.currentScreenshot); + console.log('RecorderService.mouseHandleEvent() - Screen analysis:', analysis); + const element = this.findElementAtPosition(analysis, event.x, event.y); + console.log('RecorderService.mouseHandleEvent() - Found element:', element); if (element) { this.events.push({ @@ -206,17 +229,22 @@ export class RecorderService { } private findElementAtPosition(analysis: ScreenAnalysis, x: number, y: number) { - //@ts-nocheck + console.log('RecorderService.findElementAtPosition()', { x, y, analysisElementsCount: analysis.elements.length }); return analysis.elements.find((element) => { const bounds = element.bounds; - return x >= bounds.x && + const found = x >= bounds.x && x <= bounds.x + bounds.width && y >= bounds.y && y <= bounds.y + bounds.height; + if (found) { + console.log('RecorderService.findElementAtPosition() - Found matching element:', element); + } + return found; }); } private generateBasicCode(): string { + console.log('RecorderService.generateBasicCode()', { eventsCount: this.events.length }); let basicCode = '10 REM BotDesktop Automation Script\n'; let lineNumber = 20; @@ -228,9 +256,6 @@ export class RecorderService { case 'click': basicCode += `${lineNumber} CLICK "${event.identifier}"\n`; break; - case 'type': - basicCode += `${lineNumber} TYPE "${event.identifier}"\n`; - break; case 'type': basicCode += `${lineNumber} TYPE "${event.identifier}" "${event.value}"\n`; break; @@ -242,6 +267,7 @@ export class RecorderService { } basicCode += `${lineNumber} END\n`; + console.log('RecorderService.generateBasicCode() - Generated code:', basicCode); return basicCode; } -} +} \ No newline at end of file diff --git a/src/services/types.ts b/src/services/types.ts index d4886cc..97eca85 100644 --- a/src/services/types.ts +++ b/src/services/types.ts @@ -11,6 +11,8 @@ export interface AutomationAction { }; } + + export interface AutomationEvent { type: 'click' | 'type' | 'move'; identifier: string; diff --git a/webpack.config.js b/webpack.config.js index fa2b8e9..0977fa8 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,5 +1,7 @@ const path = require('path'); const HtmlWebpackPlugin = require('html-webpack-plugin'); +// webpack.config.js +const webpack = require('webpack'); module.exports = { devtool: 'source-map', @@ -22,7 +24,10 @@ module.exports = { path: path.resolve(__dirname, 'dist/renderer'), }, plugins: [ - new HtmlWebpackPlugin({ + new webpack.ProvidePlugin({ + global: 'global', // This will make global available in your bundled code + }), + new HtmlWebpackPlugin({ template: './src/renderer/index.html' }), ],