diff --git a/cexp.sh b/cexp.sh deleted file mode 100755 index c99e56b..0000000 --- a/cexp.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Remove output.txt if it exists to start fresh -rm -f output.txt - -# Find all .ts and .tsx files excluding node_modules, and concatenate filename and contents into output.txt -find . -type f \( -name "*.ts" -o -name "*.tsx" \) -not -path "*/node_modules/*" | while read -r file; do - echo -e "\n// File: $file\n" >> output.txt - cat "$file" >> output.txt -done - -echo "All TypeScript (.ts and .tsx) code has been combined into output.txt with filenames as headers, excluding node_modules" diff --git a/cimp.sh b/cimp.sh deleted file mode 100755 index b8eeb49..0000000 --- a/cimp.sh +++ /dev/null @@ -1,337 +0,0 @@ -#!/bin/bash - -# Create project directories -mkdir -p ./src/preload -mkdir -p ./src/renderer -mkdir -p ./src/services - -# Create preload.ts file -cat < ./src/preload/preload.ts -// File: ./src/preload/preload.ts - -const { ipcRenderer } = require('electron'); - -//@ts-nocheck -(window as any).myApi = { - //@ts-nocheck - sendMessage: (message: any) => { - console.log('preload.sendMessage', { message }); - ipcRenderer.send('message-from-renderer', message); - }, - //@ts-nocheck - receiveMessage: (callback: any) => { - console.log('preload.receiveMessage', { callback }); - ipcRenderer.on('message-from-main', (event, arg) => callback(arg)); - }, -}; -EOL - -# Create index.tsx file -cat < ./src/renderer/index.tsx -// File: ./src/renderer/index.tsx - -import React from 'react'; -import ReactDOM from 'react-dom/client'; -import App from '../components/App'; - -ReactDOM.createRoot( - document.getElementById('root') as HTMLElement -).render( - - - -); -EOL - -# Create player.service.ts file -cat < ./src/services/player.service.ts -// File: ./src/services/player.service.ts - -import { ipcMain } from 'electron'; -import { AutomationEvent, ScreenAnalysis } from './types'; -import { OpenAIService } from './openai.service'; - -export class PlayerService { - private openAIService: OpenAIService; - - constructor() { - console.log('PlayerService.constructor', {}); - this.openAIService = new OpenAIService(); - } - - async executeBasicCode(code: string) { - console.log('PlayerService.executeBasicCode', { code }); - const lines = code.split('\\n'); - - for (const line of lines) { - if (line.trim().startsWith('REM') || line.trim() === '') continue; - - const match = line.match(/^\\d+\\s+(\\w+)\\s+"([^"]+)"(?:\\s+"([^"]+)")?/); - if (!match) continue; - - const [_, command, identifier, value] = match; - await this.executeCommand(command, identifier, value); - await new Promise(resolve => setTimeout(resolve, 500)); - } - } - - private async executeCommand(command: string, identifier: string, value?: string) { - console.log('PlayerService.executeCommand', { command, identifier, value }); - const screenshotPath = await this.captureScreen(); - - const analysis = await this.openAIService.analyzeScreen(screenshotPath); - const element = analysis.elements.find(e => e.identifier === identifier); - - if (!element) throw new Error(\`Element not found: \${identifier}\`); - - const centerX = element.bounds.x + element.bounds.width / 2; - const centerY = element.bounds.y + element.bounds.height / 2; - - switch (command) { - case 'CLICK': - await this.simulateClick(centerX, centerY); - break; - case 'TYPE': - await this.simulateClick(centerX, centerY); - await this.simulateTyping(value || ''); - break; - } - } - - private async captureScreen(): Promise { - console.log('PlayerService.captureScreen', {}); - return new Promise((resolve, reject) => { - ipcMain.once('screen-captured', (_, screenshotPath) => { - resolve(screenshotPath); - }); - - ipcMain.emit('capture-screen'); - }); - } - - private async simulateClick(x: number, y: number): Promise { - console.log('PlayerService.simulateClick', { x, y }); - return new Promise((resolve) => { - ipcMain.once('click-completed', () => { - resolve(); - }); - - ipcMain.emit('simulate-click', { x, y }); - }); - } - - private async simulateTyping(text: string): Promise { - console.log('PlayerService.simulateTyping', { text }); - return new Promise((resolve) => { - ipcMain.once('typing-completed', () => { - resolve(); - }); - - ipcMain.emit('simulate-typing', { text }); - }); - } -} -EOL - -# Create types.ts file -cat < ./src/services/types.ts -// File: ./src/services/types.ts - -export interface AutomationAction { - type: 'click' | 'type' | 'move'; - identifier: string; - value?: string; - confidence: number; - bounds: { - x: number; - y: number; - width: number; - height: number; - }; -} - -export interface AutomationEvent { - type: 'click' | 'type' | 'move'; - identifier: string; - value?: string; - timestamp: number; - narration: string; -} - -export interface WhisperResponse { - text: string; - segments: any; -} - -export interface ScreenContext { - screenshot: string; - transcription: string; - cursorPosition: { x: number, y: number }; -} - -export interface ScreenAnalysis { - timestamp: number, - elements: { - identifier: string; - type: string; - bounds: { x: number; y: number; width: number; height: number }; - value?: string; - }[]; -} -EOL - -# Create recorder.service.ts file -cat < ./src/services/recorder.service.ts -// File: ./src/services/recorder.service.ts - -const { ipcRenderer } = require('electron'); // Require ipcRender -import { AutomationEvent, ScreenAnalysis, WhisperResponse } from '../services/types'; -import { OpenAIService } from '../services/openai.service'; -import * as path from 'path'; -import * as fs from 'fs'; - -export class RecorderService { - private events: AutomationEvent[] = []; - private recording: boolean = false; - private openAIService: OpenAIService; - private currentScreenshot: string = ''; - private lastTranscription: string = ''; - private recordingProcess: any = null; - private tempDir: string; - private currentAudioFile: string = ''; - private silenceTimer: NodeJS.Timeout | null = null; - private isProcessingAudio: boolean = false; - - constructor() { - console.log('RecorderService.constructor', {}); - this.openAIService = new OpenAIService(); - this.tempDir = path.join(process.cwd(), 'temp_recordings'); - if (!fs.existsSync(this.tempDir)) { - fs.mkdirSync(this.tempDir, { recursive: true }); - } - } - - public async startRecording() { - console.log('RecorderService.startRecording', {}); - try { - this.recording = true; - this.events = []; - await this.setupAudioRecording(); - await this.requestScreenshot(); - ipcRenderer.on('keyboard-event', this.keyboardHandleEvent); - } catch (error) { - console.error('Failed to start recording:', error); - this.recording = false; - throw error; - } - } - - private async setupAudioRecording() { - console.log('RecorderService.setupAudioRecording', {}); - try { - ipcRenderer.on('audio-level', this.handleAudioLevel); - ipcRenderer.on('audio-chunk', this.handleAudioChunk); - } catch (error) { - console.error('Error setting up audio recording:', error); - throw new Error(\`Failed to setup audio recording: \${error.message}\`); - } - } - - private handleAudioLevel = async (_: any, level: number) => { - console.log('RecorderService.handleAudioLevel', { level }); - if (!this.recording) return; - - const SILENCE_THRESHOLD = 0.01; - const SILENCE_DURATION = 1000; - - if (level < SILENCE_THRESHOLD) { - if (!this.silenceTimer && !this.isProcessingAudio) { - this.silenceTimer = setTimeout(async () => { - if (this.recording) { - await this.processSilence(); - } - }, SILENCE_DURATION); - } - } else { - if (this.silenceTimer) { - clearTimeout(this.silenceTimer); - this.silenceTimer = null; - } - } - } - - private handleAudioChunk = async (_: any, chunk: Buffer) => { - console.log('RecorderService.handleAudioChunk', { chunk }); - if (!this.recording) return; - - try { - const audioFilePath = path.join(this.tempDir, \`audio-\${Date.now()}.wav\`); - fs.writeFileSync(audioFilePath, chunk); - - if (this.silenceTimer) { - clearTimeout(this.silenceTimer); - this.silenceTimer = null; - await this.processAudioFile(audioFilePath); - } - } catch (error) { - console.error('Error handling audio chunk:', error); - } - }; - - private async processSilence() { - console.log('RecorderService.processSilence', {}); - if (this.isProcessingAudio) return; - - this.isProcessingAudio = true; - try { - const audioFilePath = await ipcRenderer.invoke('save-audio-chunk'); - if (audioFilePath) { - this.currentAudioFile = audioFilePath; - await this.processAudioFile(audioFilePath); - await this.requestScreenshot(); - } - } catch (error) { - console.error('Error processing silence:', error); - } finally { - this.isProcessingAudio = false; - } - } - - private async processAudioFile(audioFilePath: string) { - console.log('RecorderService.processAudioFile', { audioFilePath }); - const transcription = await this.openAIService.transcribeAudio(audioFilePath); - this.lastTranscription = transcription; - await this.requestScreenshot(); - } - - private async requestScreenshot() { - console.log('RecorderService.requestScreenshot', {}); - await ipcRenderer.invoke('request-screenshot'); - } - - private keyboardHandleEvent = async (_: any, event: any) => { - console.log('RecorderService.keyboardHandleEvent', { event }); - if (!this.recording) return; - - const automationEvent: AutomationEvent = { - type: 'keyboard', - identifier: event.key, - timestamp: Date.now(), - narration: this.lastTranscription, - }; - - this.events.push(automationEvent); - }; - - public async stopRecording() { - console.log('RecorderService.stopRecording', {}); - try { - this.recording = false; - ipcRenderer.removeListener('keyboard-event', this.keyboardHandleEvent); - await ipcRenderer.invoke('stop-audio-recording'); - } catch (error) { - console.error('Failed to stop recording:', error); - } - } -} -EOL