GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

new(all): Removal of OneDrive dependencies.

Browse source
This commit is contained in:
Rodrigo Rodriguez (Pragmatismo) 2025-03-05 22:09:05 -03:00
parent e557eaa24f
commit 5fe45a04f7
12 changed files with 214 additions and 164 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -175,6 +175,7 @@
"mammoth": "1.8.0",
"mariadb": "3.3.1",
"mime-types": "2.1.35",
"minio": "^8.0.4",
"moment": "2.30.1",
"ms-rest-azure": "3.0.2",
"mysql": "^2.18.1",

2
packages/basic.gblib/services/DialogKeywords.ts
View file

@ -1495,7 +1495,7 @@ export class DialogKeywords {
else {
if (GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') === 'legacy') {
const ext = path.extname(filename);
const gbaiName = GBUtil.getGBAIPath(min.botId);

2
packages/basic.gblib/services/GBVMService.ts
View file

@ -150,7 +150,7 @@ export class GBVMService extends GBService {
let mainName = GBVMService.getMethodNameFromVBSFilename(filename);
min.scriptMap[filename] = mainName;
if (writeVBS && GBConfigService.get('STORAGE_NAME')) {
if (writeVBS && GBConfigService.get('GB_MODE') === 'legacy') {
let text = await this.getTextFromWord(folder, wordFile);
// Write VBS file without pragma keywords.

2
packages/basic.gblib/services/SystemKeywords.ts
View file

@ -2726,7 +2726,7 @@ export class SystemKeywords {
try {
let data;
if (GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') === 'legacy') {
let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min);
const gbaiName = GBUtil.getGBAIPath(min.botId);
let packagePath = '/' + urlJoin(gbaiName, `${min.botId}.gbdrive`);

2
packages/core.gbapp/services/GBConversationalService.ts
View file

@ -1280,7 +1280,7 @@ export class GBConversationalService {
* Sends a message in a user with an already started conversation (got ConversationReference set)
*/
public async sendOnConversation(min: GBMinInstance, user: GuaribasUser, message: any) {
if (GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') === 'legacy') {
const ref = JSON.parse(user.conversationReference);
MicrosoftAppCredentials.trustServiceUrl(ref.serviceUrl);
await min.bot['continueConversation'](ref, async t1 => {

156
packages/core.gbapp/services/GBCoreService.ts
View file

@ -67,6 +67,7 @@ import { GBDeployer } from './GBDeployer.js';
import { SystemKeywords } from '../../basic.gblib/services/SystemKeywords.js';
import csvdb from 'csv-database';
import { SaaSPackage } from '../../saas.gbapp/index.js';
import { Client } from 'minio';
/**
* GBCoreService contains main logic for handling storage services related
@ -120,66 +121,90 @@ export class GBCoreService implements IGBCoreService {
*/
public async initStorage(): Promise<any> {
this.dialect = GBConfigService.get('STORAGE_DIALECT');
let port: number | undefined;
let host: string | undefined;
let database: string | undefined;
let username: string | undefined;
let password: string | undefined;
let storage: string | undefined;
if (this.dialect === 'mssql') {
// Validar o dialeto
if (!['mssql', 'postgres', 'sqlite'].includes(this.dialect)) {
throw new Error(`Unknown or unsupported dialect: ${this.dialect}.`);
}
// Configurações específicas para cada dialeto
if (this.dialect === 'mssql' || this.dialect === 'postgres') {
host = GBConfigService.get('STORAGE_SERVER');
database = GBConfigService.get('STORAGE_NAME');
username = GBConfigService.get('STORAGE_USERNAME');
password = GBConfigService.get('STORAGE_PASSWORD');
const portStr = GBConfigService.get('STORAGE_PORT');
port = portStr ? parseInt(portStr, 10) : undefined;
if (!host || !database || !username || !password || !port) {
throw new Error(`Missing required configuration for ${this.dialect}.`);
}
} else if (this.dialect === 'sqlite') {
storage = GBConfigService.get('STORAGE_FILE');
if (!storage) {
throw new Error('STORAGE_FILE is required for SQLite.');
}
if (!(await GBUtil.exists(storage))) {
process.env.STORAGE_SYNC = 'true';
}
} else {
throw new Error(`Unknown dialect: ${this.dialect}.`);
}
// Configuração de logging
const logging: boolean | Function =
GBConfigService.get('STORAGE_LOGGING') === 'true'
? (str: string): void => {
GBLogEx.info(0, str);
}
: false;
// Configuração de encrypt (específico para MSSQL)
const encrypt: boolean = GBConfigService.get('STORAGE_ENCRYPT') === 'true';
const acquire = parseInt(GBConfigService.get('STORAGE_ACQUIRE_TIMEOUT'));
// Configuração do pool
const acquireStr = GBConfigService.get('STORAGE_ACQUIRE_TIMEOUT');
const acquire = acquireStr ? parseInt(acquireStr, 10) : 10000; // Valor padrão de 10 segundos
// Configuração do Sequelize
const sequelizeOptions: SequelizeOptions = {
define: {
freezeTableName: true,
timestamps: false
timestamps: false,
},
host: host,
port: port,
logging: logging as boolean,
dialect: this.dialect as Dialect,
storage: storage,
quoteIdentifiers: false, // set case-insensitive
dialectOptions: {
quoteIdentifiers: this.dialect === 'postgres',
dialectOptions: this.dialect === 'mssql' ? {
options: {
trustServerCertificate: true,
encrypt: encrypt
}
},
encrypt: encrypt,
},
} : {},
pool: {
max: 5,
min: 0,
idle: 10000,
evict: 10000,
acquire: acquire
}
acquire: acquire,
},
};
// Inicializar o Sequelize
this.sequelize = new Sequelize(database, username, password, sequelizeOptions);
}
/**
* Checks wheather storage is acessible or not and opens firewall
* in case of any connection block.
@ -675,7 +700,7 @@ await fs.writeFile('.env', env);
}
public async setConfig(min, name: string, value: any): Promise<any> {
if (GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') === 'legacy') {
// Handles calls for BASIC persistence on sheet files.
GBLog.info(`Defining Config.xlsx variable ${name}= '${value}'...`);
@ -836,29 +861,53 @@ await fs.writeFile('.env', env);
}
public async ensureFolders(instances, deployer: GBDeployer) {
const storageMode = process.env.GB_MODE;
let libraryPath = GBConfigService.get('STORAGE_LIBRARY');
if (storageMode === 'gbcluster') {
const minioClient = new Client({
endPoint: process.env.DRIVE_SERVER,
port: parseInt(process.env.DRIVE_PORT),
useSSL: process.env.DRIVE_USE_SSL === 'true',
accessKey: process.env.DRIVE_ACCESSKEY,
secretKey: process.env.DRIVE_SECRET,
});
await this.syncBotStorage(instances, 'default', deployer, libraryPath);
const bucketStream = await minioClient.listBuckets();
for await (const bucket of bucketStream) {
if (bucket.name.endsWith('.gbai') && bucket.name.startsWith(process.env.DRIVE_ORG_PREFIX)) {
if (!(await GBUtil.exists(libraryPath))) {
mkdirp.sync(libraryPath);
}
await this.syncBotStorage(instances, 'default', deployer, libraryPath);
const files = await fs.readdir(libraryPath);
await CollectionUtil.asyncForEach(files, async file => {
if (file.trim().toLowerCase() !== 'default.gbai' && file.charAt(0) !== '_') {
let botId = file.replace(/\.gbai/, '');
await this.syncBotStorage(instances, botId, deployer, libraryPath);
const botId = bucket.name.replace('.gbai', '').replace(process.env.DRIVE_ORG_PREFIX, '');
await this.syncBotStorage(instances, botId, deployer, libraryPath);
}
}
});
} else {
if (!(await GBUtil.exists(libraryPath))) {
mkdirp.sync(libraryPath);
}
await this.syncBotStorage(instances, 'default', deployer, libraryPath);
const files = await fs.readdir(libraryPath);
await CollectionUtil.asyncForEach(files, async (file) => {
if (file.trim().toLowerCase() !== 'default.gbai' && file.charAt(0) !== '_') {
let botId = file.replace(/\.gbai/, '');
await this.syncBotStorage(instances, botId, deployer, libraryPath);
}
});
}
}
private async syncBotStorage(instances: any, botId: any, deployer: GBDeployer, libraryPath: string) {
let instance = instances.find(p => p.botId.toLowerCase().trim() === botId.toLowerCase().trim());
if (!instance) {
GBLog.info(`Importing package ${botId}...`);
GBLog.info(`Importing package ${botId}.gbai...`);
// Creates a bot.
@ -866,45 +915,14 @@ await fs.writeFile('.env', env);
email = null;
instance = await deployer.deployBlankBot(botId, mobile, email);
instances.push(instance);
const gbaiPath = path.join(libraryPath, `${botId}.gbai`);
if (!(await GBUtil.exists(gbaiPath))) {
fs.mkdir(gbaiPath, { recursive: true });
const base = path.join(process.env.PWD, 'templates', 'default.gbai');
fs.cp(path.join(base, `default.gbkb`), path.join(gbaiPath, `default.gbkb`), {
errorOnExist: false,
force: true,
recursive: true
});
fs.cp(path.join(base, `default.gbot`), path.join(gbaiPath, `default.gbot`), {
errorOnExist: false,
force: true,
recursive: true
});
fs.cp(path.join(base, `default.gbtheme`), path.join(gbaiPath, `default.gbtheme`), {
errorOnExist: false,
force: true,
recursive: true
});
fs.cp(path.join(base, `default.gbdata`), path.join(gbaiPath, `default.gbdata`), {
errorOnExist: false,
force: true,
recursive: true
});
fs.cp(path.join(base, `default.gbdialog`), path.join(gbaiPath, `default.gbdialog`), {
errorOnExist: false,
force: true,
recursive: true
});
fs.cp(path.join(base, `default.gbdrive`), path.join(gbaiPath, `default.gbdrive`), {
errorOnExist: false,
force: true,
recursive: true
});
}
}
return instance;
}
public static async createWebDavServer(minInstances: GBMinInstance[]) {

170
packages/core.gbapp/services/GBDeployer.ts
View file

@ -39,6 +39,8 @@ import express from 'express';
import child_process from 'child_process';
import { rimraf } from 'rimraf';
import urlJoin from 'url-join';
import { Client } from 'minio';
import fs from 'fs/promises';
import { GBError, GBLog, GBMinInstance, IGBCoreService, IGBDeployer, IGBInstance, IGBPackage } from 'botlib';
import { AzureSearch } from 'pragmatismo-io-framework';
@ -220,7 +222,7 @@ export class GBDeployer implements IGBDeployer {
const instance = await this.importer.createBotInstance(botId);
const bootInstance = GBServer.globals.bootInstance;
if (GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') === 'legacy') {
// Gets the access token to perform service operations.
const accessToken = await (GBServer.globals.minBoot.adminService as any)['acquireElevatedToken'](
@ -232,6 +234,7 @@ export class GBDeployer implements IGBDeployer {
const service = await AzureDeployerService.createInstance(this);
const application = await service.createApplication(accessToken, botId);
// Fills new instance base information and get App secret.
instance.marketplaceId = (application as any).appId;
@ -249,7 +252,7 @@ export class GBDeployer implements IGBDeployer {
// Saves bot information to the store.
await this.core.saveInstance(instance);
if (GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') === 'legacy') {
await this.deployBotOnAzure(instance, GBServer.globals.publicAddress);
}
@ -487,6 +490,7 @@ export class GBDeployer implements IGBDeployer {
/**
*/
public async downloadFolder(
min: GBMinInstance,
localPath: string,
@ -494,86 +498,118 @@ export class GBDeployer implements IGBDeployer {
baseUrl: string = null,
client = null
): Promise<any> {
GBLogEx.info(min, `downloadFolder: localPath=${localPath}, remotePath=${remotePath}, baseUrl=${baseUrl}`);
if (!baseUrl) {
let { baseUrl, client } = await GBDeployer.internalGetDriveClient(min);
remotePath = remotePath.replace(/\\/gi, '/');
const parts = remotePath.split('/');
// Creates each subfolder.
let pathBase = localPath;
if (!(await GBUtil.exists(pathBase))) {
fs.mkdir(pathBase);
}
await CollectionUtil.asyncForEach(parts, async item => {
pathBase = path.join(pathBase, item);
if (!(await GBUtil.exists(pathBase))) {
fs.mkdir(pathBase);
}
const storageMode = process.env.GB_MODE;
if (storageMode === 'gbcluster') {
const minioClient = new Client({
endPoint: process.env.DRIVE_SERVER || 'localhost',
port: parseInt(process.env.DRIVE_PORT || '9000', 10),
useSSL: process.env.DRIVE_USE_SSL === 'true',
accessKey: process.env.DRIVE_ACCESSKEY,
secretKey: process.env.DRIVE_SECRET,
});
// Retrieves all files in remote folder.
let packagePath = GBUtil.getGBAIPath(min.botId);
packagePath = urlJoin(packagePath, remotePath);
let url = `${baseUrl}/drive/root:/${packagePath}:/children`;
GBLogEx.info(min, `Downloading: ${url}`);
let documents;
try {
const res = await client.api(url).get();
documents = res.value;
} catch (error) {
GBLogEx.info(min, `Error downloading: ${error.toString()}`);
const bucketName = process.env.DRIVE_BUCKETPREFIX + min.botId + '.gbai';
if (!(await GBUtil.exists(localPath))) {
await fs.mkdir(localPath, { recursive: true });
}
if (documents === undefined || documents.length === 0) {
GBLogEx.info(min, `${remotePath} is an empty folder.`);
return null;
}
// Download files or navigate to directory to recurse.
await CollectionUtil.asyncForEach(documents, async item => {
const itemPath = path.join(localPath, remotePath, item.name);
if (item.folder) {
const objectsStream = minioClient.listObjects(bucketName, remotePath, true);
for await (const obj of objectsStream) {
const itemPath = path.join(localPath, obj.name);
if (obj.name.endsWith('/')) {
if (!(await GBUtil.exists(itemPath))) {
fs.mkdir(itemPath);
await fs.mkdir(itemPath, { recursive: true });
}
const nextFolder = urlJoin(remotePath, item.name);
await this.downloadFolder(min, localPath, nextFolder);
} else {
let download = true;
if (await GBUtil.exists(itemPath)) {
const dt = await fs.stat(itemPath);
if (new Date(dt.mtime) >= new Date(item.lastModifiedDateTime)) {
const stats = await fs.stat(itemPath);
if (stats.mtime >= new Date(obj.lastModified)) {
download = false;
}
}
if (download) {
GBLogEx.info(min, `Downloading: ${itemPath}...`);
const url = item['@microsoft.graph.downloadUrl'];
const response = await fetch(url);
await fs.writeFile(itemPath, Buffer.from(await response.arrayBuffer()), { encoding: null });
fs.utimes(itemPath, new Date(), new Date(item.lastModifiedDateTime));
} else {
GBLogEx.info(min, `Local is up to date: ${path.basename(itemPath)}...`);
await minioClient.fGetObject(bucketName, obj.name, itemPath);
await fs.utimes(itemPath, new Date(), new Date(obj.lastModified));
}
}
});
}
} else {
if (!baseUrl) {
const { baseUrl, client } = await GBDeployer.internalGetDriveClient(min);
remotePath = remotePath.replace(/\\/gi, '/');
const parts = remotePath.split('/');
let pathBase = localPath;
if (!(await GBUtil.exists(pathBase))) {
await fs.mkdir(pathBase, { recursive: true });
}
await CollectionUtil.asyncForEach(parts, async (item) => {
pathBase = path.join(pathBase, item);
if (!(await GBUtil.exists(pathBase))) {
await fs.mkdir(pathBase, { recursive: true });
}
});
let packagePath = GBUtil.getGBAIPath(min.botId);
packagePath = urlJoin(packagePath, remotePath);
let url = `${baseUrl}/drive/root:/${packagePath}:/children`;
let documents;
try {
const res = await client.api(url).get();
documents = res.value;
} catch (error) {
GBLogEx.info(min, `Error downloading: ${error.toString()}`);
}
if (documents === undefined || documents.length === 0) {
return null;
}
await CollectionUtil.asyncForEach(documents, async (item) => {
const itemPath = path.join(localPath, remotePath, item.name);
if (item.folder) {
if (!(await GBUtil.exists(itemPath))) {
await fs.mkdir(itemPath, { recursive: true });
}
const nextFolder = urlJoin(remotePath, item.name);
await this.downloadFolder(min, localPath, nextFolder);
} else {
let download = true;
if (await GBUtil.exists(itemPath)) {
const stats = await fs.stat(itemPath);
if (new Date(stats.mtime) >= new Date(item.lastModifiedDateTime)) {
download = false;
}
}
if (download) {
const url = item['@microsoft.graph.downloadUrl'];
const response = await fetch(url);
await fs.writeFile(itemPath, new Uint8Array(await response.arrayBuffer()), { encoding: null });
await fs.utimes(itemPath, new Date(), new Date(item.lastModifiedDateTime));
}
}
});
}
}
}
/**
* UndDeploys a bot to the storage.
* Undeploys a bot to the storage.
*/
public async undeployBot(botId: string, packageName: string): Promise<void> {
// Deletes Bot registration on cloud.
@ -621,7 +657,7 @@ export class GBDeployer implements IGBDeployer {
await this.cleanupPackage(min.instance, packageName);
}
if (!GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') !== 'legacy') {
const filePath = path.join(GBConfigService.get('STORAGE_LIBRARY'), gbai, packageName);
await GBUtil.copyIfNewerRecursive(filePath, packageWorkFolder);
} else {

10
packages/core.gbapp/services/GBMinService.ts
View file

@ -730,7 +730,7 @@ export class GBMinService {
color2: this.core.getParam(instance, 'Color2', null)
};
if (!GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') !== 'legacy') {
config['domain'] = `http://localhost:${GBConfigService.get('PORT')}/directline/${botId}`;
} else {
const webchatTokenContainer = await this.getWebchatToken(instance);
@ -802,7 +802,7 @@ export class GBMinService {
? instance.marketplacePassword
: GBConfigService.get('MARKETPLACE_SECRET')
};
if (!GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') !== 'legacy') {
startRouter(GBServer.globals.server, instance.botId);
config['clientOptions'] = { baseUri: `http://localhost:${GBConfigService.get('PORT')}` };
}
@ -1047,7 +1047,7 @@ export class GBMinService {
const sec = new SecService();
let member = context.activity.recipient;
if (process.env.STORAGE_NAME || !member) {
if (process.env.GB_MODE === 'legacy' || !member) {
member = context.activity.from;
}
let user = await sec.ensureUser(min, member.id, member.name, '', 'web', member.name, null);
@ -1256,7 +1256,7 @@ export class GBMinService {
};
try {
if (!GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') !== 'legacy') {
const context = adapter['createContext'](req);
context['_activity'] = context.activity.body;
await handler(context);
@ -1800,7 +1800,7 @@ export class GBMinService {
private mutex: Mutex = new Mutex();
public async watchPackages(min: GBMinInstance, packageType: string): Promise<void> {
if (!GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') !== 'legacy') {
const packagePath = GBUtil.getGBAIPath(min.botId, packageType);
const libraryPath = path.join(GBConfigService.get('STORAGE_LIBRARY'), packagePath);

2
packages/kb.gbapp/services/KBService.ts
View file

@ -1456,7 +1456,7 @@ export class KBService implements IGBKBService {
await this.importKbPackage(min, localPath, p, instance);
GBDeployer.mountGBKBAssets(packageName, min.botId, localPath);
if (GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') === 'legacy') {
const service = await AzureDeployerService.createInstance(deployer);
const searchIndex = instance.searchIndex ? instance.searchIndex : GBServer.globals.minBoot.instance.searchIndex;
await deployer.rebuildIndex(instance, service.getKBSearchSchema(searchIndex));

2
packages/whatsapp.gblib/services/WhatsappDirectLine.ts
View file

@ -557,7 +557,7 @@ export class WhatsappDirectLine extends GBService {
WhatsappDirectLine.pidByNumber[from] = pid;
GBLogEx.info(this.min, `GBWhatsapp: Starting new conversation on Bot (pid: ${pid}).`);
let response;
if (GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') === 'legacy') {
response = await client.apis.Conversations.Conversations_StartConversation(
);

27
src/app.ts
View file

@ -186,12 +186,8 @@ export class GBServer {
// Creates a boot instance or load it from storage.
if (GBConfigService.get('STORAGE_SERVER')) {
azureDeployer = await AzureDeployerService.createInstance(deployer);
if (GBConfigService.get('GB_MODE') === 'legacy') {
await core.initStorage();
} else if (!GBConfigService.get('STORAGE_NAME')) {
await core.initStorage();
} else {
[GBServer.globals.bootInstance, azureDeployer] = await core['createBootInstanceEx'](
core,
null,
@ -201,6 +197,9 @@ export class GBServer {
);
await core.saveInstance(GBServer.globals.bootInstance);
}
else {
await core.initStorage();
}
// Deploys system and user packages.
@ -225,7 +224,7 @@ export class GBServer {
);
if (instances.length === 0) {
if (GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') === 'legacy') {
const instance = await importer.importIfNotExistsBotPackage(
GBConfigService.get('BOT_ID'),
'boot.gbot',
@ -251,7 +250,8 @@ export class GBServer {
// Just sync if not using LOAD_ONLY.
if (!GBConfigService.get('STORAGE_NAME') && !process.env.LOAD_ONLY) {
if (GBConfigService.get('GB_MODE') !== 'legacy'
&& !process.env.LOAD_ONLY) {
await core['ensureFolders'](instances, deployer);
}
GBServer.globals.bootInstance = instances[0];
@ -289,7 +289,7 @@ export class GBServer {
return proxy.web(req, res, { target: 'http://localhost:1111' }); // Express server
} else if (host === process.env.ROUTER_1) {
return proxy.web(req, res, { target: `http://localhost:${process.env.ROUTER_1_PORT}` });
} else if (host === process.env.ROUTER_2) {
return proxy.web(req, res, { target: `http://localhost:${process.env.ROUTER_2_PORT}` });
} else {
@ -308,7 +308,7 @@ export class GBServer {
core.openBrowserInDevelopment();
}
} catch (error) {
GBLog.error(`STOP: ${GBUtil.toYAML(error.message)}`);
GBLog.error(`STOP: ${GBUtil.toYAML(error)}`);
shutdown();
}
})();
@ -378,15 +378,10 @@ export class GBServer {
}
function shutdown() {
GBServer.globals.server.close(() => {
GBLogEx.info(0, 'General Bots server closed.');
GBServer.globals.apiServer.close(() => {
GBLogEx.info(0, 'General Bots API server closed.');
process.exit(0);
});
GBLogEx.info(0, 'General Bots server is now shutdown.');
});
process.exit(0);
}

2
src/util.ts
View file

@ -82,7 +82,7 @@ export class GBUtil {
*/
public static async getDirectLineClient(min: any): Promise<SwaggerClient> {
let config;
if (!GBConfigService.get('STORAGE_NAME')) {
if (GBConfigService.get('GB_MODE') !== 'legacy') {
config = {
spec: JSON.parse(await fs.readFile('directline-v2.json', 'utf8')),
requestInterceptor: req => {