GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

fix (templates): ai-search OK. Image by AI.

Browse source
This commit is contained in:
Rodrigo Rodriguez 2024-09-19 09:17:30 -03:00
parent 22fcf8b541
commit 1bae1ed5e9
9 changed files with 127 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

95
packages/basic.gblib/services/ImageProcessingServices.ts
View file

@ -39,6 +39,9 @@ import urlJoin from 'url-join';
import { GBServer } from '../../../src/app.js';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
import { GBUtil } from '../../../src/util.js';
import fs from 'fs/promises';
import { AzureOpenAI } from 'openai';
import { OpenAIClient } from '@langchain/openai';
/**
* Image processing services of conversation to be called by BASIC.
@ -61,28 +64,31 @@ export class ImageProcessingServices {
/**
* SET ORIENTATION VERTICAL
*
* file = MERGE file1, file2, file3
*
* file = MERGE file1, file2, file3
*/
public async mergeImage({pid, files})
{
public async mergeImage({ pid, files }) {
const { min, user } = await DialogKeywords.getProcessInfo(pid);
let paths = [];
await CollectionUtil.asyncForEach(files, async file => {
const gbfile = DialogKeywords.getFileByHandle(file);
const gbfile = DialogKeywords.getFileByHandle(file);
paths.push(gbfile.path);
});
const botId = min.instance.botId;
const packagePath = GBUtil.getGBAIPath(min.botId);
// TODO: const img = await joinImages(paths);
const localName = path.join('work', packagePath, 'cache', `img-mrg${GBAdminService.getRndReadableIdentifier()}.png`);
const localName = path.join(
'work',
packagePath,
'cache',
`img-mrg${GBAdminService.getRndReadableIdentifier()}.png`
);
const url = urlJoin(GBServer.globals.publicAddress, min.botId, 'cache', path.basename(localName));
// img.toFile(localName);
// img.toFile(localName);
return { localName: localName, url: url, data: null };
}
/**
@ -90,7 +96,7 @@ export class ImageProcessingServices {
*
* @example file = BLUR file
*/
public async blur({ pid, file: file }) {
public async blur({ pid, file: file }) {
const { min, user } = await DialogKeywords.getProcessInfo(pid);
GBLogEx.info(min, `Image Processing SHARPEN ${file}.`);
@ -98,4 +104,75 @@ export class ImageProcessingServices {
return;
}
public async getImageFromPrompt({ pid, prompt }) {
const { min, user, params } = await DialogKeywords.getProcessInfo(pid);
const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null, true);
const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null, true);
const azureOpenAIVersion = await (min.core as any)['getParam'](min.instance, 'Azure Open AI Version', null, true);
const azureOpenAIImageModel = await (min.core as any)['getParam'](min.instance, 'Azure Open AI Image Model', null, true);
if (azureOpenAIKey) {
// Initialize the Azure OpenAI client
const client = new AzureOpenAI({
baseURL: azureOpenAIEndpoint,
apiVersion: azureOpenAIVersion,
apiKey: azureOpenAIKey
});
// Make a request to the image generation endpoint
const response = await client.images.generate({
model: azureOpenAIImageModel,
prompt: prompt,
n: 1,
size: '1024x1024'
});
const gbaiName = GBUtil.getGBAIPath(min.botId);
const localName = path.join('work', gbaiName, 'cache', `DALL-E${GBAdminService.getRndReadableIdentifier()}.png`);
const url = response.data[0].url;
const res = await fetch(url);
let buf: any = Buffer.from(await res.arrayBuffer());
await fs.writeFile(localName, buf, { encoding: null });
GBLogEx.info(min, `DALL-E: ${url} - ${response.data[0].revised_prompt}.`);
return { localName, url };
}
}
public async getCaptionForImage({ pid, imageUrl }) {
const { min, user, params } = await DialogKeywords.getProcessInfo(pid);
const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null);
const azureOpenAITextModel = 'gpt-4'; // Specify GPT-4 model here
const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null);
const azureOpenAIVersion = await (min.core as any)['getParam'](min.instance, 'Azure Open AI Version', null, true);
if (azureOpenAIKey && azureOpenAITextModel && imageUrl) {
const client = new AzureOpenAI({
apiVersion: azureOpenAIVersion,
apiKey: azureOpenAIKey,
baseURL: azureOpenAIEndpoint
});
const prompt = `Provide a descriptive caption for the image at the following URL: ${imageUrl}`;
const response = await client.completions.create({
model: azureOpenAITextModel,
prompt: prompt,
max_tokens: 50
});
const caption = response['data'].choices[0].text.trim();
GBLogEx.info(min, `Generated caption: ${caption}`);
return { caption };
}
}
}

11
packages/basic.gblib/services/KeywordsExpressions.ts
View file

@ -484,11 +484,20 @@ export class KeywordsExpressions {
keywords[i++] = [
/^\s*(.*)\=\s*(GET IMAGE)(\s*)(.*)/gim,
($0, $1, $2, $3, $4) => {
const params = this.getParams($4, ['text']);
const params = this.getParams($4, ['prompt']);
return `${$1} = await img.getImageFromPrompt({pid: pid, ${params}})`;
}
];
keywords[i++] = [
/\s*(PLAY)(\s*)(.*)/gim,
($0, $1, $2, $3, $4) => {
const params = this.getParams($3, ['file']);
return `await sys.play({pid: pid, ${params}})`;
}
];
keywords[i++] = [
/^\s*(.*)\=\s*(AUTO SAVE)(\s*)(.*)/gim,
($0, $1, $2, $3, $4) => {

14
packages/basic.gblib/services/SystemKeywords.ts
View file

@ -2822,15 +2822,15 @@ export class SystemKeywords {
async bail => {
if (table.endsWith('.csv')) {
// CSV handling
const packagePath = GBUtil.getGBAIPath(min.botId, "gbdata");
const packagePath = GBUtil.getGBAIPath(min.botId, 'gbdata');
const csvFile = path.join(GBConfigService.get('STORAGE_LIBRARY'), packagePath, `${table}`);
try {
// Try to read the file to get headers
const data = await fs.readFile(csvFile, 'utf8');
const headers = data.split('\n')[0].split(',');
const db = await csvdb(csvFile, headers, ',');
// Append new row
await db.add(dst);
item = dst;
@ -2860,4 +2860,12 @@ export class SystemKeywords {
);
return item;
}
public async play({ pid, file }) {
const { min, user, params, step } = await DialogKeywords.getProcessInfo(pid);
await min.kbService.playUrl(min, min.conversationalService, step, file?.url ? file.url : file);
await this.setMemoryContext({ pid, erase: true });
}
}

15
packages/default.gbui/public/index.html
View file

@ -58,13 +58,14 @@
</style>
<script>
// Focus the input field after rendering
setTimeout(() => {
const input = document.querySelector('.webchat__send-box-text-box__input');
if (input) {
input.focus();
}
}, 3000); // Adjust timing as needed
document.addEventListener('DOMContentLoaded', () => {
setTimeout(() => {
const input = document.querySelector('.webchat__send-box-text-box__input');
if (input) {
input.focus();
}
}, 3000); // Adjust timing as needed
});
</script>
</head>

2
packages/kb.gbapp/services/KBService.ts
View file

@ -1383,7 +1383,7 @@ export class KBService implements IGBKBService {
conversationalService.sendAudio(min, step, answer.content);
}
private async playUrl(
public async playUrl(
min,
conversationalService: IGBConversationalService,
step: GBDialogStep,

72
packages/llm.gblib/services/ImageServices.ts
View file

@ -30,82 +30,10 @@
'use strict';
import { GBMinInstance } from 'botlib';
import OpenAI from 'openai';
import { AzureKeyCredential } from '@azure/core-auth';
import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords';
import path from 'path';
import { GBServer } from '../../../src/app.js';
import fs from 'fs/promises';
import urlJoin from 'url-join';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx';
import { GBUtil } from '../../../src/util';
/**
* Image processing services of conversation to be called by BASIC.
*/
export class ImageServices {
public async getImageFromPrompt({ pid, prompt }) {
const { min, user, params } = await DialogKeywords.getProcessInfo(pid);
const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null);
const azureOpenAIImageModel = await min.core.getParam(min.instance, 'Azure Open Image Model', null);
const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null);
if (azureOpenAIKey) {
// Initialize the Azure OpenAI client
const client = new OpenAI({ apiKey: azureOpenAIKey, baseURL: azureOpenAIEndpoint });
// Make a request to the image generation endpoint
const response = await client.images.generate({
prompt: prompt,
n: 1,
size: '1024x1024'
});
const gbaiName = GBUtil.getGBAIPath(min.botId);
const localName = path.join('work', gbaiName, 'cache', `DALL-E${GBAdminService.getRndReadableIdentifier()}.png`);
const url = response.data[0].url;
const res = await fetch(url);
let buf: any = Buffer.from(await res.arrayBuffer());
await fs.writeFile(localName, buf, { encoding: null });
GBLogEx.info(min, `DALL-E image generated at ${url}.`);
return { localName, url };
}
}
public async getCaptionForImage({ pid, imageUrl }) {
const { min, user, params } = await DialogKeywords.getProcessInfo(pid);
const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null);
const azureOpenAITextModel = 'gpt-4'; // Specify GPT-4 model here
const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null);
if (azureOpenAIKey && azureOpenAITextModel && imageUrl) {
// Initialize the Azure OpenAI client
const client = new OpenAI({ apiKey: azureOpenAIKey, baseURL: azureOpenAIEndpoint });
// Construct a prompt to describe the image and generate a caption
const prompt = `Provide a descriptive caption for the image at the following URL: ${imageUrl}`;
// Generate a caption using GPT-4
const response = await client.completions.create({
model: azureOpenAITextModel,
prompt: prompt,
max_tokens: 50
});
const caption = response['data'].choices[0].text.trim();
GBLogEx.info(min, `Generated caption: ${caption}`);
return { caption };
}
}
}

7
templates/ai-search.gbai/ai-search.gbdialog/start.bas Normal file
View file

@ -0,0 +1,7 @@
image = GET IMAGE "AI Search BOT and write General Bots."
PLAY image
BEGIN SYSTEM PROMPT
END SYSTEM PROMPT

2
templates/ai-search.gbai/ai-search.gbot/config.csv
View file

@ -1,3 +1,3 @@
name,value
Answer Mode,document-ref
Theme Color,violet
Theme Color,blue

1 name value
2 Answer Mode document-ref
3 Theme Color violet blue

6
templates/default.gbai/default.gbtheme/css/App.css
View file

@ -186,9 +186,9 @@ screen and (min-width: 420px) and (max-width: 1000px) {
@media screen and (min-width: 1000px) {
.player {
width: 45% !important;
height: 71% !important;
border: 7px solid var(--color-light-border);
width: 46% !important;
height: 73% !important;
border: 8px solid var(--color-light-border);
position: absolute;
left: 3%;
top: 24%;