fix (templates): ai-search OK. Image by AI.

packages/basic.gblib/services/ImageProcessingServices.ts

@@ -39,6 +39,9 @@ import urlJoin from 'url-join';
 import { GBServer } from '../../../src/app.js';
 import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
 import { GBUtil } from '../../../src/util.js';
+import fs from 'fs/promises';
+import { AzureOpenAI } from 'openai';
+import { OpenAIClient } from '@langchain/openai';
 
 /**
  * Image processing services of conversation to be called by BASIC.
@@ -64,8 +67,7 @@ export class ImageProcessingServices {
    *
    * file = MERGE file1, file2, file3
    */
-  public async mergeImage({pid, files})
+  public async mergeImage({ pid, files }) {
-  {
     const { min, user } = await DialogKeywords.getProcessInfo(pid);
 
     let paths = [];
@@ -77,12 +79,16 @@ export class ImageProcessingServices {
     const botId = min.instance.botId;
     const packagePath = GBUtil.getGBAIPath(min.botId);
     // TODO: const img = await joinImages(paths);
-    const localName = path.join('work', packagePath, 'cache', `img-mrg${GBAdminService.getRndReadableIdentifier()}.png`);
+    const localName = path.join(
+      'work',
+      packagePath,
+      'cache',
+      `img-mrg${GBAdminService.getRndReadableIdentifier()}.png`
+    );
     const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
-//    img.toFile(localName);
+    //    img.toFile(localName);
 
     return { localName: localName, url: url, data: null };
-
   }
 
   /**
@@ -98,4 +104,75 @@ export class ImageProcessingServices {
     return;
   }
 
+  public async getImageFromPrompt({ pid, prompt }) {
+    const { min, user, params } = await DialogKeywords.getProcessInfo(pid);
+
+    const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null, true);
+    const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null, true);
+    const azureOpenAIVersion = await (min.core as any)['getParam'](min.instance, 'Azure Open AI Version', null, true);
+    const azureOpenAIImageModel = await (min.core as any)['getParam'](min.instance, 'Azure Open AI Image Model', null, true);
+    
+
+    if (azureOpenAIKey) {
+      // Initialize the Azure OpenAI client
+
+      const client = new AzureOpenAI({
+        baseURL: azureOpenAIEndpoint,
+        apiVersion: azureOpenAIVersion,
+        apiKey: azureOpenAIKey
+      });
+      
+      // Make a request to the image generation endpoint
+      
+      const response = await client.images.generate({
+        model: azureOpenAIImageModel,
+        prompt: prompt,
+        n: 1,
+        size: '1024x1024'
+      });
+
+      const gbaiName = GBUtil.getGBAIPath(min.botId);
+      const localName = path.join('work', gbaiName, 'cache', `DALL-E${GBAdminService.getRndReadableIdentifier()}.png`);
+
+      const url = response.data[0].url;
+      const res = await fetch(url);
+      let buf: any = Buffer.from(await res.arrayBuffer());
+      await fs.writeFile(localName, buf, { encoding: null });
+
+      GBLogEx.info(min, `DALL-E: ${url} - ${response.data[0].revised_prompt}.`);
+
+      return { localName, url };
+    }
+  }
+
+  public async getCaptionForImage({ pid, imageUrl }) {
+    const { min, user, params } = await DialogKeywords.getProcessInfo(pid);
+
+    const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null);
+    const azureOpenAITextModel = 'gpt-4'; // Specify GPT-4 model here
+    const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null);
+    const azureOpenAIVersion = await (min.core as any)['getParam'](min.instance, 'Azure Open AI Version', null, true);
+
+    if (azureOpenAIKey && azureOpenAITextModel && imageUrl) {
+      const client = new AzureOpenAI({
+        apiVersion: azureOpenAIVersion,
+        apiKey: azureOpenAIKey,
+        baseURL: azureOpenAIEndpoint
+      });
+
+      const prompt = `Provide a descriptive caption for the image at the following URL: ${imageUrl}`;
+
+      const response = await client.completions.create({
+
+        model: azureOpenAITextModel,
+        prompt: prompt,
+        max_tokens: 50
+      });
+
+      const caption = response['data'].choices[0].text.trim();
+      GBLogEx.info(min, `Generated caption: ${caption}`);
+
+      return { caption };
+    }
+  }
 }

packages/basic.gblib/services/KeywordsExpressions.ts

@@ -484,11 +484,20 @@ export class KeywordsExpressions {
     keywords[i++] = [
       /^\s*(.*)\=\s*(GET IMAGE)(\s*)(.*)/gim,
       ($0, $1, $2, $3, $4) => {
-        const params = this.getParams($4, ['text']);
+        const params = this.getParams($4, ['prompt']);
         return `${$1} = await img.getImageFromPrompt({pid: pid, ${params}})`;
       }
     ];
 
+    keywords[i++] = [
+      /\s*(PLAY)(\s*)(.*)/gim,
+      ($0, $1, $2, $3, $4) => {
+        const params = this.getParams($3, ['file']);
+        return `await sys.play({pid: pid, ${params}})`;
+      }
+    ];
+
+
     keywords[i++] = [
       /^\s*(.*)\=\s*(AUTO SAVE)(\s*)(.*)/gim,
       ($0, $1, $2, $3, $4) => {

packages/basic.gblib/services/SystemKeywords.ts

@@ -2822,7 +2822,7 @@ export class SystemKeywords {
       async bail => {
         if (table.endsWith('.csv')) {
           // CSV handling
-          const packagePath = GBUtil.getGBAIPath(min.botId, "gbdata");
+          const packagePath = GBUtil.getGBAIPath(min.botId, 'gbdata');
           const csvFile = path.join(GBConfigService.get('STORAGE_LIBRARY'), packagePath, `${table}`);
 
           try {
@@ -2860,4 +2860,12 @@ export class SystemKeywords {
     );
     return item;
   }
+
+  public async play({ pid, file }) {
+    const { min, user, params, step } = await DialogKeywords.getProcessInfo(pid);
+
+    await min.kbService.playUrl(min, min.conversationalService, step, file?.url ? file.url : file);
+
+    await this.setMemoryContext({ pid, erase: true });
+  }
 }

packages/default.gbui/public/index.html

@@ -58,13 +58,14 @@
     </style>
 
     <script>
-      // Focus the input field after rendering
+      document.addEventListener('DOMContentLoaded', () => {
         setTimeout(() => {
           const input = document.querySelector('.webchat__send-box-text-box__input');
           if (input) {
             input.focus();
           }
         }, 3000); // Adjust timing as needed
+      });
     </script>
   </head>
 

packages/kb.gbapp/services/KBService.ts

@@ -1383,7 +1383,7 @@ export class KBService implements IGBKBService {
     conversationalService.sendAudio(min, step, answer.content);
   }
 
-  private async playUrl(
+  public async playUrl(
     min,
     conversationalService: IGBConversationalService,
     step: GBDialogStep,

packages/llm.gblib/services/ImageServices.ts

@@ -30,82 +30,10 @@
 
 'use strict';
 
-import { GBMinInstance } from 'botlib';
-import OpenAI from 'openai';
-
-import { AzureKeyCredential } from '@azure/core-auth';
-import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords';
-import path from 'path';
-import { GBServer } from '../../../src/app.js';
-import fs from 'fs/promises'; 
-import urlJoin from 'url-join';
-import { GBAdminService } from '../../admin.gbapp/services/GBAdminService';
-import { GBLogEx } from '../../core.gbapp/services/GBLogEx';
-import { GBUtil } from '../../../src/util';
 
 /**
  * Image processing services of conversation to be called by BASIC.
  */
 export class ImageServices {
-  public async getImageFromPrompt({ pid, prompt }) {
-    const { min, user, params } = await DialogKeywords.getProcessInfo(pid);
 
-    const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null);
-    const azureOpenAIImageModel = await min.core.getParam(min.instance, 'Azure Open Image Model', null);
-    const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null);
-
-    if (azureOpenAIKey) {
-      // Initialize the Azure OpenAI client
-
-      const client = new OpenAI({ apiKey: azureOpenAIKey, baseURL: azureOpenAIEndpoint });
-
-      // Make a request to the image generation endpoint
-
-      const response = await client.images.generate({
-        prompt: prompt,
-        n: 1,
-        size: '1024x1024'
-      });
-
-      const gbaiName = GBUtil.getGBAIPath(min.botId);
-      const localName = path.join('work', gbaiName, 'cache', `DALL-E${GBAdminService.getRndReadableIdentifier()}.png`);
-
-      const url = response.data[0].url;
-      const res = await fetch(url);
-      let buf: any = Buffer.from(await res.arrayBuffer());
-  await fs.writeFile(localName, buf, { encoding: null });
-
-      GBLogEx.info(min, `DALL-E image generated at ${url}.`);
-
-      return { localName, url };
-    }
-  }
-
-  public async getCaptionForImage({ pid, imageUrl }) {
-    const { min, user, params } = await DialogKeywords.getProcessInfo(pid);
-
-    const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null);
-    const azureOpenAITextModel = 'gpt-4'; // Specify GPT-4 model here
-    const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null);
-
-    if (azureOpenAIKey && azureOpenAITextModel && imageUrl) {
-      // Initialize the Azure OpenAI client
-      const client = new OpenAI({ apiKey: azureOpenAIKey, baseURL: azureOpenAIEndpoint });
-
-      // Construct a prompt to describe the image and generate a caption
-      const prompt = `Provide a descriptive caption for the image at the following URL: ${imageUrl}`;
-
-      // Generate a caption using GPT-4
-      const response = await client.completions.create({
-        model: azureOpenAITextModel,
-        prompt: prompt,
-        max_tokens: 50
-      });
-
-      const caption = response['data'].choices[0].text.trim();
-      GBLogEx.info(min, `Generated caption: ${caption}`);
-
-      return { caption };
-    }
-  }
 }

templates/ai-search.gbai/ai-search.gbdialog/start.bas

@@ -0,0 +1,7 @@
+image = GET IMAGE "AI Search BOT and write General Bots."
+
+PLAY image 
+
+BEGIN SYSTEM PROMPT
+
+END SYSTEM PROMPT 

templates/ai-search.gbai/ai-search.gbot/config.csv

@@ -1,3 +1,3 @@
 name,value
 Answer Mode,document-ref
-Theme Color,violet
+Theme Color,blue

templates/default.gbai/default.gbtheme/css/App.css

@@ -186,9 +186,9 @@ screen and (min-width: 420px) and (max-width: 1000px) {
 
 @media screen and (min-width: 1000px) {
     .player {
-        width: 45% !important;
+        width: 46% !important;
-        height: 71% !important;
+        height: 73% !important;
-        border: 7px solid var(--color-light-border);
+        border: 8px solid var(--color-light-border);
         position: absolute;
         left: 3%;
         top: 24%;