#!/bin/bash # Create project directories mkdir -p ./src/preload mkdir -p ./src/renderer mkdir -p ./src/services # Create preload.ts file cat < ./src/preload/preload.ts // File: ./src/preload/preload.ts const { ipcRenderer } = require('electron'); //@ts-nocheck (window as any).myApi = { //@ts-nocheck sendMessage: (message: any) => { console.log('preload.sendMessage', { message }); ipcRenderer.send('message-from-renderer', message); }, //@ts-nocheck receiveMessage: (callback: any) => { console.log('preload.receiveMessage', { callback }); ipcRenderer.on('message-from-main', (event, arg) => callback(arg)); }, }; EOL # Create index.tsx file cat < ./src/renderer/index.tsx // File: ./src/renderer/index.tsx import React from 'react'; import ReactDOM from 'react-dom/client'; import App from '../components/App'; ReactDOM.createRoot( document.getElementById('root') as HTMLElement ).render( ); EOL # Create player.service.ts file cat < ./src/services/player.service.ts // File: ./src/services/player.service.ts import { ipcMain } from 'electron'; import { AutomationEvent, ScreenAnalysis } from './types'; import { OpenAIService } from './openai.service'; export class PlayerService { private openAIService: OpenAIService; constructor() { console.log('PlayerService.constructor', {}); this.openAIService = new OpenAIService(); } async executeBasicCode(code: string) { console.log('PlayerService.executeBasicCode', { code }); const lines = code.split('\\n'); for (const line of lines) { if (line.trim().startsWith('REM') || line.trim() === '') continue; const match = line.match(/^\\d+\\s+(\\w+)\\s+"([^"]+)"(?:\\s+"([^"]+)")?/); if (!match) continue; const [_, command, identifier, value] = match; await this.executeCommand(command, identifier, value); await new Promise(resolve => setTimeout(resolve, 500)); } } private async executeCommand(command: string, identifier: string, value?: string) { console.log('PlayerService.executeCommand', { command, identifier, value }); const screenshotPath = await this.captureScreen(); const analysis = await this.openAIService.analyzeScreen(screenshotPath); const element = analysis.elements.find(e => e.identifier === identifier); if (!element) throw new Error(\`Element not found: \${identifier}\`); const centerX = element.bounds.x + element.bounds.width / 2; const centerY = element.bounds.y + element.bounds.height / 2; switch (command) { case 'CLICK': await this.simulateClick(centerX, centerY); break; case 'TYPE': await this.simulateClick(centerX, centerY); await this.simulateTyping(value || ''); break; } } private async captureScreen(): Promise { console.log('PlayerService.captureScreen', {}); return new Promise((resolve, reject) => { ipcMain.once('screen-captured', (_, screenshotPath) => { resolve(screenshotPath); }); ipcMain.emit('capture-screen'); }); } private async simulateClick(x: number, y: number): Promise { console.log('PlayerService.simulateClick', { x, y }); return new Promise((resolve) => { ipcMain.once('click-completed', () => { resolve(); }); ipcMain.emit('simulate-click', { x, y }); }); } private async simulateTyping(text: string): Promise { console.log('PlayerService.simulateTyping', { text }); return new Promise((resolve) => { ipcMain.once('typing-completed', () => { resolve(); }); ipcMain.emit('simulate-typing', { text }); }); } } EOL # Create types.ts file cat < ./src/services/types.ts // File: ./src/services/types.ts export interface AutomationAction { type: 'click' | 'type' | 'move'; identifier: string; value?: string; confidence: number; bounds: { x: number; y: number; width: number; height: number; }; } export interface AutomationEvent { type: 'click' | 'type' | 'move'; identifier: string; value?: string; timestamp: number; narration: string; } export interface WhisperResponse { text: string; segments: any; } export interface ScreenContext { screenshot: string; transcription: string; cursorPosition: { x: number, y: number }; } export interface ScreenAnalysis { timestamp: number, elements: { identifier: string; type: string; bounds: { x: number; y: number; width: number; height: number }; value?: string; }[]; } EOL # Create recorder.service.ts file cat < ./src/services/recorder.service.ts // File: ./src/services/recorder.service.ts const { ipcRenderer } = require('electron'); // Require ipcRender import { AutomationEvent, ScreenAnalysis, WhisperResponse } from '../services/types'; import { OpenAIService } from '../services/openai.service'; import * as path from 'path'; import * as fs from 'fs'; export class RecorderService { private events: AutomationEvent[] = []; private recording: boolean = false; private openAIService: OpenAIService; private currentScreenshot: string = ''; private lastTranscription: string = ''; private recordingProcess: any = null; private tempDir: string; private currentAudioFile: string = ''; private silenceTimer: NodeJS.Timeout | null = null; private isProcessingAudio: boolean = false; constructor() { console.log('RecorderService.constructor', {}); this.openAIService = new OpenAIService(); this.tempDir = path.join(process.cwd(), 'temp_recordings'); if (!fs.existsSync(this.tempDir)) { fs.mkdirSync(this.tempDir, { recursive: true }); } } public async startRecording() { console.log('RecorderService.startRecording', {}); try { this.recording = true; this.events = []; await this.setupAudioRecording(); await this.requestScreenshot(); ipcRenderer.on('keyboard-event', this.keyboardHandleEvent); } catch (error) { console.error('Failed to start recording:', error); this.recording = false; throw error; } } private async setupAudioRecording() { console.log('RecorderService.setupAudioRecording', {}); try { ipcRenderer.on('audio-level', this.handleAudioLevel); ipcRenderer.on('audio-chunk', this.handleAudioChunk); } catch (error) { console.error('Error setting up audio recording:', error); throw new Error(\`Failed to setup audio recording: \${error.message}\`); } } private handleAudioLevel = async (_: any, level: number) => { console.log('RecorderService.handleAudioLevel', { level }); if (!this.recording) return; const SILENCE_THRESHOLD = 0.01; const SILENCE_DURATION = 1000; if (level < SILENCE_THRESHOLD) { if (!this.silenceTimer && !this.isProcessingAudio) { this.silenceTimer = setTimeout(async () => { if (this.recording) { await this.processSilence(); } }, SILENCE_DURATION); } } else { if (this.silenceTimer) { clearTimeout(this.silenceTimer); this.silenceTimer = null; } } } private handleAudioChunk = async (_: any, chunk: Buffer) => { console.log('RecorderService.handleAudioChunk', { chunk }); if (!this.recording) return; try { const audioFilePath = path.join(this.tempDir, \`audio-\${Date.now()}.wav\`); fs.writeFileSync(audioFilePath, chunk); if (this.silenceTimer) { clearTimeout(this.silenceTimer); this.silenceTimer = null; await this.processAudioFile(audioFilePath); } } catch (error) { console.error('Error handling audio chunk:', error); } }; private async processSilence() { console.log('RecorderService.processSilence', {}); if (this.isProcessingAudio) return; this.isProcessingAudio = true; try { const audioFilePath = await ipcRenderer.invoke('save-audio-chunk'); if (audioFilePath) { this.currentAudioFile = audioFilePath; await this.processAudioFile(audioFilePath); await this.requestScreenshot(); } } catch (error) { console.error('Error processing silence:', error); } finally { this.isProcessingAudio = false; } } private async processAudioFile(audioFilePath: string) { console.log('RecorderService.processAudioFile', { audioFilePath }); const transcription = await this.openAIService.transcribeAudio(audioFilePath); this.lastTranscription = transcription; await this.requestScreenshot(); } private async requestScreenshot() { console.log('RecorderService.requestScreenshot', {}); await ipcRenderer.invoke('request-screenshot'); } private keyboardHandleEvent = async (_: any, event: any) => { console.log('RecorderService.keyboardHandleEvent', { event }); if (!this.recording) return; const automationEvent: AutomationEvent = { type: 'keyboard', identifier: event.key, timestamp: Date.now(), narration: this.lastTranscription, }; this.events.push(automationEvent); }; public async stopRecording() { console.log('RecorderService.stopRecording', {}); try { this.recording = false; ipcRenderer.removeListener('keyboard-event', this.keyboardHandleEvent); await ipcRenderer.invoke('stop-audio-recording'); } catch (error) { console.error('Failed to stop recording:', error); } } } EOL