diff --git a/packages/basic.gblib/services/ImageProcessingServices.ts b/packages/basic.gblib/services/ImageProcessingServices.ts index c3758e8c..10a21fbf 100644 --- a/packages/basic.gblib/services/ImageProcessingServices.ts +++ b/packages/basic.gblib/services/ImageProcessingServices.ts @@ -39,6 +39,9 @@ import urlJoin from 'url-join'; import { GBServer } from '../../../src/app.js'; import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js'; import { GBUtil } from '../../../src/util.js'; +import fs from 'fs/promises'; +import { AzureOpenAI } from 'openai'; +import { OpenAIClient } from '@langchain/openai'; /** * Image processing services of conversation to be called by BASIC. @@ -61,28 +64,31 @@ export class ImageProcessingServices { /** * SET ORIENTATION VERTICAL - * - * file = MERGE file1, file2, file3 + * + * file = MERGE file1, file2, file3 */ - public async mergeImage({pid, files}) - { + public async mergeImage({ pid, files }) { const { min, user } = await DialogKeywords.getProcessInfo(pid); let paths = []; await CollectionUtil.asyncForEach(files, async file => { - const gbfile = DialogKeywords.getFileByHandle(file); + const gbfile = DialogKeywords.getFileByHandle(file); paths.push(gbfile.path); }); const botId = min.instance.botId; const packagePath = GBUtil.getGBAIPath(min.botId); // TODO: const img = await joinImages(paths); - const localName = path.join('work', packagePath, 'cache', `img-mrg${GBAdminService.getRndReadableIdentifier()}.png`); + const localName = path.join( + 'work', + packagePath, + 'cache', + `img-mrg${GBAdminService.getRndReadableIdentifier()}.png` + ); const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName)); -// img.toFile(localName); + // img.toFile(localName); return { localName: localName, url: url, data: null }; - } /** @@ -90,7 +96,7 @@ export class ImageProcessingServices { * * @example file = BLUR file */ - public async blur({ pid, file: file }) { + public async blur({ pid, file: file }) { const { min, user } = await DialogKeywords.getProcessInfo(pid); GBLogEx.info(min, `Image Processing SHARPEN ${file}.`); @@ -98,4 +104,75 @@ export class ImageProcessingServices { return; } + public async getImageFromPrompt({ pid, prompt }) { + const { min, user, params } = await DialogKeywords.getProcessInfo(pid); + + const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null, true); + const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null, true); + const azureOpenAIVersion = await (min.core as any)['getParam'](min.instance, 'Azure Open AI Version', null, true); + const azureOpenAIImageModel = await (min.core as any)['getParam'](min.instance, 'Azure Open AI Image Model', null, true); + + + if (azureOpenAIKey) { + // Initialize the Azure OpenAI client + + const client = new AzureOpenAI({ + baseURL: azureOpenAIEndpoint, + apiVersion: azureOpenAIVersion, + apiKey: azureOpenAIKey + }); + + // Make a request to the image generation endpoint + + const response = await client.images.generate({ + model: azureOpenAIImageModel, + prompt: prompt, + n: 1, + size: '1024x1024' + }); + + const gbaiName = GBUtil.getGBAIPath(min.botId); + const localName = path.join('work', gbaiName, 'cache', `DALL-E${GBAdminService.getRndReadableIdentifier()}.png`); + + const url = response.data[0].url; + const res = await fetch(url); + let buf: any = Buffer.from(await res.arrayBuffer()); + await fs.writeFile(localName, buf, { encoding: null }); + + GBLogEx.info(min, `DALL-E: ${url} - ${response.data[0].revised_prompt}.`); + + return { localName, url }; + } + } + + public async getCaptionForImage({ pid, imageUrl }) { + const { min, user, params } = await DialogKeywords.getProcessInfo(pid); + + const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null); + const azureOpenAITextModel = 'gpt-4'; // Specify GPT-4 model here + const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null); + const azureOpenAIVersion = await (min.core as any)['getParam'](min.instance, 'Azure Open AI Version', null, true); + + if (azureOpenAIKey && azureOpenAITextModel && imageUrl) { + const client = new AzureOpenAI({ + apiVersion: azureOpenAIVersion, + apiKey: azureOpenAIKey, + baseURL: azureOpenAIEndpoint + }); + + const prompt = `Provide a descriptive caption for the image at the following URL: ${imageUrl}`; + + const response = await client.completions.create({ + + model: azureOpenAITextModel, + prompt: prompt, + max_tokens: 50 + }); + + const caption = response['data'].choices[0].text.trim(); + GBLogEx.info(min, `Generated caption: ${caption}`); + + return { caption }; + } + } } diff --git a/packages/basic.gblib/services/KeywordsExpressions.ts b/packages/basic.gblib/services/KeywordsExpressions.ts index f0e3cabe..896fa9ae 100644 --- a/packages/basic.gblib/services/KeywordsExpressions.ts +++ b/packages/basic.gblib/services/KeywordsExpressions.ts @@ -484,11 +484,20 @@ export class KeywordsExpressions { keywords[i++] = [ /^\s*(.*)\=\s*(GET IMAGE)(\s*)(.*)/gim, ($0, $1, $2, $3, $4) => { - const params = this.getParams($4, ['text']); + const params = this.getParams($4, ['prompt']); return `${$1} = await img.getImageFromPrompt({pid: pid, ${params}})`; } ]; + keywords[i++] = [ + /\s*(PLAY)(\s*)(.*)/gim, + ($0, $1, $2, $3, $4) => { + const params = this.getParams($3, ['file']); + return `await sys.play({pid: pid, ${params}})`; + } + ]; + + keywords[i++] = [ /^\s*(.*)\=\s*(AUTO SAVE)(\s*)(.*)/gim, ($0, $1, $2, $3, $4) => { diff --git a/packages/basic.gblib/services/SystemKeywords.ts b/packages/basic.gblib/services/SystemKeywords.ts index e67100ce..8c17cbfd 100644 --- a/packages/basic.gblib/services/SystemKeywords.ts +++ b/packages/basic.gblib/services/SystemKeywords.ts @@ -2822,15 +2822,15 @@ export class SystemKeywords { async bail => { if (table.endsWith('.csv')) { // CSV handling - const packagePath = GBUtil.getGBAIPath(min.botId, "gbdata"); + const packagePath = GBUtil.getGBAIPath(min.botId, 'gbdata'); const csvFile = path.join(GBConfigService.get('STORAGE_LIBRARY'), packagePath, `${table}`); - + try { // Try to read the file to get headers const data = await fs.readFile(csvFile, 'utf8'); const headers = data.split('\n')[0].split(','); const db = await csvdb(csvFile, headers, ','); - + // Append new row await db.add(dst); item = dst; @@ -2860,4 +2860,12 @@ export class SystemKeywords { ); return item; } + + public async play({ pid, file }) { + const { min, user, params, step } = await DialogKeywords.getProcessInfo(pid); + + await min.kbService.playUrl(min, min.conversationalService, step, file?.url ? file.url : file); + + await this.setMemoryContext({ pid, erase: true }); + } } diff --git a/packages/default.gbui/public/index.html b/packages/default.gbui/public/index.html index df7a1ff4..193958a5 100644 --- a/packages/default.gbui/public/index.html +++ b/packages/default.gbui/public/index.html @@ -58,13 +58,14 @@ diff --git a/packages/kb.gbapp/services/KBService.ts b/packages/kb.gbapp/services/KBService.ts index f240c469..7700a545 100644 --- a/packages/kb.gbapp/services/KBService.ts +++ b/packages/kb.gbapp/services/KBService.ts @@ -1383,7 +1383,7 @@ export class KBService implements IGBKBService { conversationalService.sendAudio(min, step, answer.content); } - private async playUrl( + public async playUrl( min, conversationalService: IGBConversationalService, step: GBDialogStep, diff --git a/packages/llm.gblib/services/ImageServices.ts b/packages/llm.gblib/services/ImageServices.ts index d149337e..21072373 100644 --- a/packages/llm.gblib/services/ImageServices.ts +++ b/packages/llm.gblib/services/ImageServices.ts @@ -30,82 +30,10 @@ 'use strict'; -import { GBMinInstance } from 'botlib'; -import OpenAI from 'openai'; - -import { AzureKeyCredential } from '@azure/core-auth'; -import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords'; -import path from 'path'; -import { GBServer } from '../../../src/app.js'; -import fs from 'fs/promises'; -import urlJoin from 'url-join'; -import { GBAdminService } from '../../admin.gbapp/services/GBAdminService'; -import { GBLogEx } from '../../core.gbapp/services/GBLogEx'; -import { GBUtil } from '../../../src/util'; /** * Image processing services of conversation to be called by BASIC. */ export class ImageServices { - public async getImageFromPrompt({ pid, prompt }) { - const { min, user, params } = await DialogKeywords.getProcessInfo(pid); - const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null); - const azureOpenAIImageModel = await min.core.getParam(min.instance, 'Azure Open Image Model', null); - const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null); - - if (azureOpenAIKey) { - // Initialize the Azure OpenAI client - - const client = new OpenAI({ apiKey: azureOpenAIKey, baseURL: azureOpenAIEndpoint }); - - // Make a request to the image generation endpoint - - const response = await client.images.generate({ - prompt: prompt, - n: 1, - size: '1024x1024' - }); - - const gbaiName = GBUtil.getGBAIPath(min.botId); - const localName = path.join('work', gbaiName, 'cache', `DALL-E${GBAdminService.getRndReadableIdentifier()}.png`); - - const url = response.data[0].url; - const res = await fetch(url); - let buf: any = Buffer.from(await res.arrayBuffer()); - await fs.writeFile(localName, buf, { encoding: null }); - - GBLogEx.info(min, `DALL-E image generated at ${url}.`); - - return { localName, url }; - } - } - - public async getCaptionForImage({ pid, imageUrl }) { - const { min, user, params } = await DialogKeywords.getProcessInfo(pid); - - const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null); - const azureOpenAITextModel = 'gpt-4'; // Specify GPT-4 model here - const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null); - - if (azureOpenAIKey && azureOpenAITextModel && imageUrl) { - // Initialize the Azure OpenAI client - const client = new OpenAI({ apiKey: azureOpenAIKey, baseURL: azureOpenAIEndpoint }); - - // Construct a prompt to describe the image and generate a caption - const prompt = `Provide a descriptive caption for the image at the following URL: ${imageUrl}`; - - // Generate a caption using GPT-4 - const response = await client.completions.create({ - model: azureOpenAITextModel, - prompt: prompt, - max_tokens: 50 - }); - - const caption = response['data'].choices[0].text.trim(); - GBLogEx.info(min, `Generated caption: ${caption}`); - - return { caption }; - } - } } diff --git a/templates/ai-search.gbai/ai-search.gbdialog/start.bas b/templates/ai-search.gbai/ai-search.gbdialog/start.bas new file mode 100644 index 00000000..62407381 --- /dev/null +++ b/templates/ai-search.gbai/ai-search.gbdialog/start.bas @@ -0,0 +1,7 @@ +image = GET IMAGE "AI Search BOT and write General Bots." + +PLAY image + +BEGIN SYSTEM PROMPT + +END SYSTEM PROMPT \ No newline at end of file diff --git a/templates/ai-search.gbai/ai-search.gbot/config.csv b/templates/ai-search.gbai/ai-search.gbot/config.csv index c5dd5ae4..bcda747e 100644 --- a/templates/ai-search.gbai/ai-search.gbot/config.csv +++ b/templates/ai-search.gbai/ai-search.gbot/config.csv @@ -1,3 +1,3 @@ name,value Answer Mode,document-ref -Theme Color,violet \ No newline at end of file +Theme Color,blue diff --git a/templates/default.gbai/default.gbtheme/css/App.css b/templates/default.gbai/default.gbtheme/css/App.css index 87e627c6..43748f56 100644 --- a/templates/default.gbai/default.gbtheme/css/App.css +++ b/templates/default.gbai/default.gbtheme/css/App.css @@ -186,9 +186,9 @@ screen and (min-width: 420px) and (max-width: 1000px) { @media screen and (min-width: 1000px) { .player { - width: 45% !important; - height: 71% !important; - border: 7px solid var(--color-light-border); + width: 46% !important; + height: 73% !important; + border: 8px solid var(--color-light-border); position: absolute; left: 3%; top: 24%;