From 6aaae55a6191bfc6bf772ada494b150e3a81cbc3 Mon Sep 17 00:00:00 2001 From: Rodrigo Rodriguez Date: Sun, 1 Sep 2024 21:46:55 -0300 Subject: [PATCH] fix(llm.gblib): Tool fix. More templates. --- packages/llm.gblib/services/ChatServices.ts | 7 +++-- packages/llm.gblib/services/ImageServices.ts | 28 +++++++++++++++++++ .../llm-tools.gbdata/products.csv | 7 +++++ ...ribe-image.bas => describe-image-tool.bas} | 5 ++-- ...rate-image.bas => generate-image-tool.bas} | 8 ++---- .../llm-tools.gbdialog/get-price.bas | 6 ++++ .../{search.bas => search-tool.bas} | 6 ++-- .../llm-tools.gbdialog/start.bas | 9 ++++++ .../llm-tools.gbai/llm-tools.gbot/config.csv | 2 ++ .../marketing.gbdialog/poster.bas | 2 +- 10 files changed, 66 insertions(+), 14 deletions(-) create mode 100644 templates/llm-tools.gbai/llm-tools.gbdata/products.csv rename templates/llm-tools.gbai/llm-tools.gbdialog/{describe-image.bas => describe-image-tool.bas} (71%) rename templates/llm-tools.gbai/llm-tools.gbdialog/{generate-image.bas => generate-image-tool.bas} (66%) create mode 100644 templates/llm-tools.gbai/llm-tools.gbdialog/get-price.bas rename templates/llm-tools.gbai/llm-tools.gbdialog/{search.bas => search-tool.bas} (65%) create mode 100644 templates/llm-tools.gbai/llm-tools.gbdialog/start.bas create mode 100644 templates/llm-tools.gbai/llm-tools.gbot/config.csv diff --git a/packages/llm.gblib/services/ChatServices.ts b/packages/llm.gblib/services/ChatServices.ts index 6d977796..59da3b27 100644 --- a/packages/llm.gblib/services/ChatServices.ts +++ b/packages/llm.gblib/services/ChatServices.ts @@ -71,7 +71,7 @@ import { SQL_MYSQL_PROMPT } from 'langchain/chains/sql_db'; import { GBUtil } from '../../../src/util.js'; - +import { z } from 'zod'; export interface CustomOutputParserFields {} export type ExpectedOutput = any; @@ -475,7 +475,7 @@ export class ChatServices { result = res.text ? res.text : res; sources = res.sources; - } else if (LLMMode === 'function') { + } else if (LLMMode === 'tool') { result = await conversationalToolChain.invoke({ question }); @@ -639,7 +639,8 @@ export class ChatServices { if (funcObj) { // TODO: Use ajv. - funcObj.schema = eval(jsonSchemaToZod(funcObj.parameters)); + + funcObj.schema = jsonSchemaToZod(funcObj.parameters); functions.push(new DynamicStructuredTool(funcObj)); } } diff --git a/packages/llm.gblib/services/ImageServices.ts b/packages/llm.gblib/services/ImageServices.ts index 57acf3fa..fc273055 100644 --- a/packages/llm.gblib/services/ImageServices.ts +++ b/packages/llm.gblib/services/ImageServices.ts @@ -79,4 +79,32 @@ export class ImageServices { return { localName, url }; } } + + public async getCaptionForImage({ pid, imageUrl }) { + const { min, user, params } = await DialogKeywords.getProcessInfo(pid); + + const azureOpenAIKey = await min.core.getParam(min.instance, 'Azure Open AI Key', null); + const azureOpenAITextModel = 'gpt-4'; // Specify GPT-4 model here + const azureOpenAIEndpoint = await min.core.getParam(min.instance, 'Azure Open AI Endpoint', null); + + if (azureOpenAIKey && azureOpenAITextModel && imageUrl) { + // Initialize the Azure OpenAI client + const client = new OpenAI({ apiKey: azureOpenAIKey, baseURL: azureOpenAIEndpoint }); + + // Construct a prompt to describe the image and generate a caption + const prompt = `Provide a descriptive caption for the image at the following URL: ${imageUrl}`; + + // Generate a caption using GPT-4 + const response = await client.completions.create({ + model: azureOpenAITextModel, + prompt: prompt, + max_tokens: 50 + }); + + const caption = response['data'].choices[0].text.trim(); + GBLogEx.info(min, `Generated caption: ${caption}`); + + return { caption }; + } + } } diff --git a/templates/llm-tools.gbai/llm-tools.gbdata/products.csv b/templates/llm-tools.gbai/llm-tools.gbdata/products.csv new file mode 100644 index 00000000..f882d14a --- /dev/null +++ b/templates/llm-tools.gbai/llm-tools.gbdata/products.csv @@ -0,0 +1,7 @@ +name,price +Product A, 230 +Product B, 120 +Product C, 180 +Product D, 250 +Product E, 200 +Product F, 150 \ No newline at end of file diff --git a/templates/llm-tools.gbai/llm-tools.gbdialog/describe-image.bas b/templates/llm-tools.gbai/llm-tools.gbdialog/describe-image-tool.bas similarity index 71% rename from templates/llm-tools.gbai/llm-tools.gbdialog/describe-image.bas rename to templates/llm-tools.gbai/llm-tools.gbdialog/describe-image-tool.bas index bb608b6b..a503b3d6 100644 --- a/templates/llm-tools.gbai/llm-tools.gbdialog/describe-image.bas +++ b/templates/llm-tools.gbai/llm-tools.gbdialog/describe-image-tool.bas @@ -1,5 +1,4 @@ -PARAM image +PARAM image AS STRING LIKE "https://server/image" DESCRIPTION Returns the description of the image that was sent with the previous user message. This tool is automatically invoked if a user uploads an image. - - +DEBUG "Hello" \ No newline at end of file diff --git a/templates/llm-tools.gbai/llm-tools.gbdialog/generate-image.bas b/templates/llm-tools.gbai/llm-tools.gbdialog/generate-image-tool.bas similarity index 66% rename from templates/llm-tools.gbai/llm-tools.gbdialog/generate-image.bas rename to templates/llm-tools.gbai/llm-tools.gbdialog/generate-image-tool.bas index ac7b3319..4ccd2737 100644 --- a/templates/llm-tools.gbai/llm-tools.gbdialog/generate-image.bas +++ b/templates/llm-tools.gbai/llm-tools.gbdialog/generate-image-tool.bas @@ -1,6 +1,4 @@ - -PARAM prompt +PARAM prompt as STRING LIKE "Generate a cat." DESCRIPTION Calls an artificial intelligence model to create an image. `prompt` parameter is a text description of the desired image. - - - +FIND +RETURN "https://" diff --git a/templates/llm-tools.gbai/llm-tools.gbdialog/get-price.bas b/templates/llm-tools.gbai/llm-tools.gbdialog/get-price.bas new file mode 100644 index 00000000..30f1363c --- /dev/null +++ b/templates/llm-tools.gbai/llm-tools.gbdialog/get-price.bas @@ -0,0 +1,6 @@ +PARAM product AS string LIKE "Product A" +DESCRIPTION "Returns the price of the given product." + +product = FIND "products.csv", "name LIKE ${product}" +price = product.price +RETURN price diff --git a/templates/llm-tools.gbai/llm-tools.gbdialog/search.bas b/templates/llm-tools.gbai/llm-tools.gbdialog/search-tool.bas similarity index 65% rename from templates/llm-tools.gbai/llm-tools.gbdialog/search.bas rename to templates/llm-tools.gbai/llm-tools.gbdialog/search-tool.bas index 270fdef3..4880e68c 100644 --- a/templates/llm-tools.gbai/llm-tools.gbdialog/search.bas +++ b/templates/llm-tools.gbai/llm-tools.gbdialog/search-tool.bas @@ -1,2 +1,4 @@ -PARAM query -DESCRIPTION Returns search results in a JSON string. `query` parameter is a well-formed web search query.- `search_web(query: str) -> str` returns Bing search results in a JSON string. `query` parameter is a well-formed web search query. \ No newline at end of file +PARAM query AS STRING LIKE "How is to code?" +DESCRIPTION Returns search results in a JSON string. `query` parameter is a well-formed web search query.- `search_web(query: str) -> str` returns Bing search results in a JSON string. `query` parameter is a well-formed web search query. + +DEBUG "{}" \ No newline at end of file diff --git a/templates/llm-tools.gbai/llm-tools.gbdialog/start.bas b/templates/llm-tools.gbai/llm-tools.gbdialog/start.bas new file mode 100644 index 00000000..c33da72d --- /dev/null +++ b/templates/llm-tools.gbai/llm-tools.gbdialog/start.bas @@ -0,0 +1,9 @@ +BEGIN SYSTEM PROMPT + +There exist some helpful predefined internal tools which can help me by +extending my functionalities or get me helpful information. +These tools **should** be abstracted away from the user. +These tools can be invoked only by me before I respond to a user. +Here is the list of my internal tools: + +END SYSTEM PROMPT \ No newline at end of file diff --git a/templates/llm-tools.gbai/llm-tools.gbot/config.csv b/templates/llm-tools.gbai/llm-tools.gbot/config.csv new file mode 100644 index 00000000..4f0821f8 --- /dev/null +++ b/templates/llm-tools.gbai/llm-tools.gbot/config.csv @@ -0,0 +1,2 @@ +name,value +Answer Mode,tool diff --git a/templates/marketing.gbai/marketing.gbdialog/poster.bas b/templates/marketing.gbai/marketing.gbdialog/poster.bas index d85f64af..025e05cc 100644 --- a/templates/marketing.gbai/marketing.gbdialog/poster.bas +++ b/templates/marketing.gbai/marketing.gbdialog/poster.bas @@ -5,7 +5,7 @@ pass = "*************" o = get "https://oooooooooo" # Criar a legenda para o post -caption = REWRITE "Crie um post sobre Hotmart e seus produtos, no estilo dica do dia, incluindo 10 hashtags, estilo Instagram o texto! Importante, retorne só a saída de texto pronta" +caption = REWRITE "Crie um post sobre produtos, no estilo dica do dia, incluindo 10 hashtags, estilo Instagram o texto! Importante, retorne só a saída de texto pronta" # Obter uma imagem relacionada ao conteúdo image = GET IMAGE caption