diff --git a/package.json b/package.json index 851d3f88..03c5d47b 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "@azure/keyvault-keys": "4.8.0", "@azure/ms-rest-js": "2.7.0", "@azure/msal-node": "2.8.1", + "@azure/openai": "^2.0.0-beta.1", "@azure/search-documents": "12.0.0", "@azure/storage-blob": "12.18.0", "@google-cloud/pubsub": "4.4.0", @@ -139,6 +140,7 @@ "html-to-md": "^0.8.5", "http-proxy": "1.18.1", "ibm-watson": "9.1.0", + "instagram-private-api": "^1.46.1", "iso-639-1": "3.1.2", "join-images-updated": "1.1.11", "js-md5": "0.8.3", @@ -192,13 +194,13 @@ "rimraf": "5.0.7", "safe-buffer": "5.2.1", "scanf": "1.2.0", - "sqlite3": "5.1.7", "sequelize": "6.28.2", "sequelize-cli": "6.6.0", "sequelize-typescript": "2.1.5", "sharp": "0.33.4", "simple-git": "3.24.0", "speakingurl": "14.0.1", + "sqlite3": "5.1.7", "ssr-for-bots": "1.0.1-c", "strict-password-generator": "1.1.2", "swagger-client": "3.28.1", diff --git a/packages/basic.gblib/services/GBVMService.ts b/packages/basic.gblib/services/GBVMService.ts index fbe097a7..e5ded8ab 100644 --- a/packages/basic.gblib/services/GBVMService.ts +++ b/packages/basic.gblib/services/GBVMService.ts @@ -542,6 +542,14 @@ export class GBVMService extends GBService { let col = 1; let index = 1; + const mid(str, start, length) { + start = start - 1; + if (length === undefined) { + return str.substring(start); + } + return str.substring(start, start + length); + } + // Makes objects in BASIC insensitive. const caseInsensitive = (listOrRow) => { diff --git a/packages/basic.gblib/services/KeywordsExpressions.ts b/packages/basic.gblib/services/KeywordsExpressions.ts index 527142ea..68275fbf 100644 --- a/packages/basic.gblib/services/KeywordsExpressions.ts +++ b/packages/basic.gblib/services/KeywordsExpressions.ts @@ -473,6 +473,14 @@ export class KeywordsExpressions { } ]; + keywords[i++] = [ + /^\s*(.*)\=\s*(GET IMAGE)(\s*)(.*)/gim, + ($0, $1, $2, $3, $4) => { + const params = this.getParams($4, ['text']); + return `${$1} = await img.getImageFromPrompt({pid: pid, ${params}})`; + } + ]; + keywords[i++] = [ /^\s*(.*)\=\s*(AUTO SAVE)(\s*)(.*)/gim, ($0, $1, $2, $3, $4) => { diff --git a/packages/basic.gblib/services/SystemKeywords.ts b/packages/basic.gblib/services/SystemKeywords.ts index 3dbdb6aa..c25b69cc 100644 --- a/packages/basic.gblib/services/SystemKeywords.ts +++ b/packages/basic.gblib/services/SystemKeywords.ts @@ -29,6 +29,9 @@ \*****************************************************************************/ 'use strict'; +import { IgApiClient } from 'instagram-private-api'; +import { readFileSync } from 'fs'; +import { resolve } from 'path'; import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs'; import { GBLog, GBMinInstance } from 'botlib'; import { GBConfigService } from '../../core.gbapp/services/GBConfigService.js'; @@ -918,10 +921,10 @@ export class SystemKeywords { await retry( async bail => { const result = await client - .api( - `${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='${address}')` - ) - .patch(body); + .api( + `${baseUrl}/drive/items/${document.id}/workbook/worksheets('${sheets.value[0].name}')/range(address='${address}')` + ) + .patch(body); if (result.status != 200) { GBLogEx.info(min, `Waiting 5 secs. before retrying HTTP ${result.status} GET: ${result.url}`); @@ -934,8 +937,8 @@ export class SystemKeywords { onRetry: err => { GBLog.error(`Retrying HTTP GET due to: ${err.message}.`); } - }); - + } + ); } /** @@ -2714,7 +2717,7 @@ export class SystemKeywords { public async setContext({ pid, text }) { const { min, user, params } = await DialogKeywords.getProcessInfo(pid); ChatServices.userSystemPrompt[user.userSystemId] = text; - + await this.setMemoryContext({ pid, erase: true }); } @@ -2744,4 +2747,19 @@ export class SystemKeywords { } ); } + + public async postToInstagram({ pid, username, password, imagePath, caption }) { + const { min, user, params } = await DialogKeywords.getProcessInfo(pid); + + const ig = new IgApiClient(); + ig.state.generateDevice(username); + await ig.account.login(username, password); + const imageBuffer = readFileSync(resolve(imagePath)); + const publishResult = await ig.publish.photo({ + file: imageBuffer, + caption + }); + + GBLogEx.info(min, `Image posted on IG: ${publishResult}`); + } } diff --git a/packages/gpt.gblib/services/ChatServices.ts b/packages/gpt.gblib/services/ChatServices.ts index c1a9bc45..0ed32fa7 100644 --- a/packages/gpt.gblib/services/ChatServices.ts +++ b/packages/gpt.gblib/services/ChatServices.ts @@ -575,6 +575,7 @@ export class ChatServices { }, { result: finalResponsePrompt.pipe(model).pipe(new StringOutputParser()), + // Pipe the query through here unchanged so it gets logged alongside the result. sql: previousStepResult => previousStepResult.query } diff --git a/packages/gpt.gblib/services/ImageServices.ts b/packages/gpt.gblib/services/ImageServices.ts index 83a17a3f..970c3241 100644 --- a/packages/gpt.gblib/services/ImageServices.ts +++ b/packages/gpt.gblib/services/ImageServices.ts @@ -32,29 +32,52 @@ 'use strict'; -// import { GBMinInstance } from 'botlib'; -// import {DallEAPIWrapper} from '@langchain/openai'; +import { GBMinInstance } from 'botlib'; +import { OpenAIClient } from '@azure/openai'; +import { AzureKeyCredential } from '@azure/core-auth'; +import { DialogKeywords } from '../../basic.gblib/services/DialogKeywords'; +import Path from 'path'; +import { GBServer } from '../../../src/app.js'; +import Fs from 'fs'; +import urlJoin from 'url-join'; +import { GBAdminService } from '../../admin.gbapp/services/GBAdminService'; +import { GBLogEx } from '../../core.gbapp/services/GBLogEx'; -// /** -// * Image processing services of conversation to be called by BASIC. -// */ -// export class ImageServices { -// public async getImageFromDescription(min: GBMinInstance, text: string): Promise { -// const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null); -// const azureOpenAIImageModel = await min.core.getParam(min.instance, 'Azure Open Image Model', null); -// const azureOpenAIVersion = await min.core.getParam(min.instance, 'Azure Open AI Version', null); -// const azureOpenAIApiInstanceName = await min.core.getParam(min.instance, 'Azure Open AI Instance', null); +/** + * Image processing services of conversation to be called by BASIC. + */ +export class ImageServices { -// if (azureOpenAIKey) { -// const tool = new DallEAPIWrapper({ -// n: 1, -// model: 'dall-e-3', -// apiKey: azureOpenAIKey -// }); + public async getImageFromPrompt({ pid, prompt }) { + const { min, user, params } = await DialogKeywords.getProcessInfo(pid); -// const imageURL = await tool.invoke('a painting of a cat'); + const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null); + const azureOpenAIImageModel = await min.core.getParam(min.instance, 'Azure Open Image Model', null); + const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null); -// return imageURL; -// } -// } -// } + if (azureOpenAIKey) { + // Initialize the Azure OpenAI client + const client = new OpenAIClient(azureOpenAIEndpoint, new AzureKeyCredential(azureOpenAIKey)); + + // Make a request to the image generation endpoint + + const response = await client.getImageGeneration(azureOpenAIImageModel, { + prompt: prompt, + n: 1, + size: '1024x1024' + }); + + const gbaiName = DialogKeywords.getGBAIPath(min.botId); + const localName = Path.join('work', gbaiName, 'cache', `img${GBAdminService.getRndReadableIdentifier()}.png`); + + const url = response.data[0].url; + const res = await fetch(url); + let buf: any = Buffer.from(await res.arrayBuffer()); + Fs.writeFileSync(localName, buf, { encoding: null }); + + GBLogEx.info(min, `BASIC: DALL-E image generated at ${url}.`); + + return {localName, url}; + } + } +}