GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

fix(basic.gblib): #282 Fix SSR for Bots 3.0.

Browse source
This commit is contained in:
rodrigorodriguez 2023-02-18 16:48:40 -03:00
parent bc85f714ca
commit 7f3bd7d8fe
7 changed files with 253 additions and 283 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
packages/basic.gblib/services/ChartServices.ts
View file

@ -32,7 +32,7 @@
'use strict';
import { createBrowser } from '../../core.gbapp/services/GBSSR.js';
import { GBSSR }from '../../core.gbapp/services/GBSSR.js';
export class ChartServices {
/**
@ -41,7 +41,7 @@ export class ChartServices {
* @param {string} path screenshot image full path with file name
*/
public static async screenshot (args, path) {
const browser = await createBrowser(null);
const browser = await GBSSR.createBrowser(null);
const page = await browser.newPage();
// load billboard.js assets from CDN.

4
packages/basic.gblib/services/SystemKeywords.ts
View file

@ -39,7 +39,7 @@ import { DialogKeywords } from './DialogKeywords.js';
import { GBServer } from '../../../src/app.js';
import { GBVMService } from './GBVMService.js';
import Fs from 'fs';
import { createBrowser } from '../../core.gbapp/services/GBSSR.js';
import { GBSSR }from '../../core.gbapp/services/GBSSR.js';
import urlJoin from 'url-join';
import Excel from 'exceljs';
import { TwitterApi } from 'twitter-api-v2';
@ -257,7 +257,7 @@ export class SystemKeywords {
const { min, user } = await DialogKeywords.getProcessInfo(pid);
const gbaiName = `${min.botId}.gbai`;
const browser = await createBrowser(null);
const browser = await GBSSR.createBrowser(null);
const page = await browser.newPage();
// Includes the associated CSS related to current theme.

39
packages/basic.gblib/services/WebAutomationServices.ts
View file

@ -32,19 +32,19 @@
'use strict';
import { GBLog, GBMinInstance } from 'botlib';
import { GBServer } from '../../../src/app.js';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
import { createBrowser } from '../../core.gbapp/services/GBSSR.js';
import { GuaribasUser } from '../../security.gbapp/models/index.js';
import { DialogKeywords } from './DialogKeywords.js';
import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
import urlJoin from 'url-join';
import Fs from 'fs';
import Path from 'path';
import url from 'url';
import { Mutex, Semaphore, withTimeout } from 'async-mutex';
import { GBLog, GBMinInstance } from 'botlib';
import { GBServer } from '../../../src/app.js';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
import { GBSSR }from '../../core.gbapp/services/GBSSR.js';
import { GuaribasUser } from '../../security.gbapp/models/index.js';
import { DialogKeywords } from './DialogKeywords.js';
import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
import { Mutex } from 'async-mutex';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
/**
@ -172,16 +172,17 @@ export class WebAutomationServices {
let browser;
if (!page) {
browser = await createBrowser(null);
browser = await GBSSR.createBrowser(null);
page = (await browser.pages())[0];
if (username || password) {
await page.authenticate({ pid, username: username, password: password });
}
}
// There is no session yet,
// There is no session yet.
if (!session && sessionKind === 'AS') {
// A new web session is being created.
handle = WebAutomationServices.cyrb53(this.min.botId + url);
@ -459,4 +460,20 @@ export class WebAutomationServices {
return file;
}
private async recursiveFindInFrames (inputFrame, selector) {
const frames = inputFrame.childFrames();
const results = await Promise.all(
frames.map(async frame => {
const el = await frame.$(selector);
if (el) return el;
if (frame.childFrames().length > 0) {
return await this.recursiveFindInFrames(frame, selector);
}
return null;
})
);
return results.find(Boolean);
}
}

16
packages/core.gbapp/services/GBMinService.ts
View file

@ -87,6 +87,7 @@ import { GoogleChatDirectLine } from '../../google-chat.gblib/services/GoogleCha
import { SystemKeywords } from '../../basic.gblib/services/SystemKeywords.js';
import * as nlp from 'node-nlp';
import Path from 'path';
import { GBSSR } from './GBSSR.js';
/**
* Minimal service layer for a bot and encapsulation of BOT Framework calls.
@ -236,6 +237,7 @@ export class GBMinService {
const url = `/api/messages/${botId}`;
removeRoute(GBServer.globals.server, url);
const uiUrl = `/${botId}`;
removeRoute(GBServer.globals.server, uiUrl);
@ -295,6 +297,10 @@ export class GBMinService {
if (!Fs.existsSync(dir)) {
mkdirp.sync(dir);
}
dir = `work/${min.botId}.gbai/${min.botId}.gbui`;
if (!Fs.existsSync(dir)) {
mkdirp.sync(dir);
}
// Loads Named Entity data for this bot.
@ -370,15 +376,17 @@ export class GBMinService {
if (process.env.DISABLE_WEB !== 'true') {
const uiUrl = `/${instance.botId}`;
let staticHandler = express.static(urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build'));
GBServer.globals.server.get(uiUrl, async (req, res, next)=> {
await GBSSR.ssrFilter(req, res, staticHandler as any);
});
const uiUrlAlt = `/${instance.activationCode}`;
GBServer.globals.server.use(
uiUrl,
express.static(urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build'))
);
GBServer.globals.server.use(
uiUrlAlt,
express.static(urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build'))
);
const domain = min.core.getParam(min.instance, 'Domain', null);
if (domain) {
GBServer.globals.server.use(

151
packages/core.gbapp/services/GBSSR.ts
View file

@ -36,27 +36,27 @@
'use strict';
import {createRequire} from "module";
const require = createRequire(import.meta.url);
const puppeteer = require('puppeteer-extra');
const hidden = require('puppeteer-extra-plugin-stealth')
// require executablePath from puppeteer
const {executablePath} = require('puppeteer')
import Path from 'path';
import Fs from 'fs';
import { NextFunction, Request, Response } from 'express';
import urljoin from 'url-join';
import { GBMinInstance } from 'botlib';
import { GBServer } from '../../../src/app.js';
import { GBLogEx } from './GBLogEx.js';
import { createRequire } from 'module';
const require = createRequire(import.meta.url);
const puppeteer = require('puppeteer-extra');
const hidden = require('puppeteer-extra-plugin-stealth');
const { executablePath } = require('puppeteer');
export class GBSSR {
// https://hackernoon.com/tips-and-tricks-for-web-scraping-with-puppeteer-ed391a63d952
// Dont download all resources, we just need the HTML
// Also, this is huge performance/response time boost
const blockedResourceTypes = ['image', 'media', 'font', 'texttrack', 'object', 'beacon', 'csp_report', 'imageset'];
private blockedResourceTypes = ['image', 'media', 'font', 'texttrack', 'object', 'beacon', 'csp_report', 'imageset'];
// const whitelist = ["document", "script", "xhr", "fetch"];
const skippedResources = [
private skippedResources = [
'quantserve',
'adzerk',
'doubleclick',
@ -78,9 +78,7 @@ const skippedResources = [
'tiqcdn'
];
const RENDER_CACHE = new Map();
async function createBrowser (profilePath): Promise<any> {
public static async createBrowser(profilePath): Promise<any> {
let args = [
'--check-for-update-interval=2592000',
'--disable-accelerated-2d-canvas',
@ -102,7 +100,7 @@ async function createBrowser (profilePath): Promise<any> {
Fs.writeFileSync(preferences, JSON.stringify(data));
}
}
puppeteer.use(hidden())
puppeteer.use(hidden());
const browser = await puppeteer.launch({
args: args,
ignoreHTTPSErrors: true,
@ -114,38 +112,12 @@ async function createBrowser (profilePath): Promise<any> {
return browser;
}
async function recursiveFindInFrames (inputFrame, selector) {
const frames = inputFrame.childFrames();
const results = await Promise.all(
frames.map(async frame => {
const el = await frame.$(selector);
if (el) return el;
if (frame.childFrames().length > 0) {
return await recursiveFindInFrames(frame, selector);
}
return null;
})
);
return results.find(Boolean);
}
/**
* https://developers.google.com/web/tools/puppeteer/articles/ssr#reuseinstance
* @param {string} url URL to prerender.
* Return the HTML of bot default.gbui.
*/
async function ssr (url: string, useCache: boolean, cacheRefreshRate: number) {
if (RENDER_CACHE.has(url) && useCache) {
const cached = RENDER_CACHE.get(url);
if (Date.now() - cached.renderedAt > cacheRefreshRate && !(cacheRefreshRate <= 0)) {
RENDER_CACHE.delete(url);
} else {
return {
html: cached.html,
status: 200
};
}
}
const browser = await createBrowser(null);
public async getHTML(min: GBMinInstance) {
const url = urljoin(GBServer.globals.publicAddress, min.botId);
const browser = await GBSSR.createBrowser(null);
const stylesheetContents = {};
try {
@ -155,13 +127,10 @@ async function ssr (url: string, useCache: boolean, cacheRefreshRate: number) {
);
await page.setRequestInterception(true);
page.on('request', request => {
const requestUrl = request
.url()
.split('?')[0]
.split('#')[0];
const requestUrl = request.url().split('?')[0].split('#')[0];
if (
blockedResourceTypes.indexOf(request.resourceType()) !== -1 ||
skippedResources.some(resource => requestUrl.indexOf(resource) !== -1)
this.blockedResourceTypes.indexOf(request.resourceType()) !== -1 ||
this.skippedResources.some(resource => requestUrl.indexOf(resource) !== -1)
) {
request.abort();
} else {
@ -191,6 +160,7 @@ async function ssr (url: string, useCache: boolean, cacheRefreshRate: number) {
await sleep(45000);
// Inject <base> on page to relative resources load properly.
await page.evaluate(url => {
const base = document.createElement('base');
base.href = url;
@ -199,6 +169,7 @@ async function ssr (url: string, useCache: boolean, cacheRefreshRate: number) {
}, url);
// Remove scripts and html imports. They've already executed.
await page.evaluate(() => {
const elements = document.querySelectorAll('script, link[rel="import"]');
elements.forEach(e => {
@ -207,6 +178,7 @@ async function ssr (url: string, useCache: boolean, cacheRefreshRate: number) {
});
// Replace stylesheets in the page with their equivalent <style>.
await page.$$eval(
'link[rel="stylesheet"]',
(links, content) => {
@ -225,48 +197,25 @@ async function ssr (url: string, useCache: boolean, cacheRefreshRate: number) {
const html = await page.content();
// Close the page we opened here (not the browser).
await page.close();
if (useCache) {
RENDER_CACHE.set(url, { html, renderedAt: Date.now() });
}
return { html, status: response!.status() };
return html;
} catch (e) {
const html = e.toString();
console.warn({ message: `URL: ${url} Failed with message: ${html}` });
return { html, status: 500 };
GBLogEx.error(min, `URL: ${url} Failed with message: ${html}`);
return html;
} finally {
await browser.close();
}
}
function clearCache () {
RENDER_CACHE.clear();
}
interface Options {
prerender?: Array<string>;
exclude?: Array<string>;
useCache?: boolean;
cacheRefreshRate?: number;
}
function ssrForBots (
options: Options = {
public static async ssrFilter(req: Request, res: Response, next) {
let applyOptions = {
prerender: [], // Array containing the user-agents that will trigger the ssr service
exclude: [], // Array containing paths and/or extentions that will be excluded from being prerendered by the ssr service
useCache: true, // Variable that determins if we will use page caching or not
cacheRefreshRate: 86400 // Seconds of which the cache will be kept alive, pass 0 or negative value for infinite lifespan
}
) {
let applyOptions = Object.assign(
{
prerender: [], // Array containing the user-agents that will trigger the ssr service
exclude: [], // Array containing paths and/or extentions that will be excluded from being prerendered by the ssr service
useCache: true, // Variable that determins if we will use page caching or not
cacheRefreshRate: 86400 // Seconds of which the cache will be kept alive, pass 0 or negative value for infinite lifespan
},
options
);
};
// Default user agents
const prerenderArray = [
@ -296,37 +245,31 @@ function ssrForBots (
// default exclude array
const excludeArray = ['.xml', '.ico', '.txt', '.json'];
function ssrOnDemand (req: Request, res: Response, next: NextFunction) {
Promise.resolve(() => {
return true;
})
.then(async () => {
const userAgent: string = req.headers['user-agent'] || '';
const prerender = new RegExp([...prerenderArray, ...applyOptions.prerender].join('|').slice(0, -1), 'i').test(
userAgent
);
const exclude = !new RegExp([...excludeArray, ...applyOptions.exclude].join('|').slice(0, -1)).test(
req.originalUrl
);
// Reads from static HTML when a bot is crawling.
const botId = req.originalUrl ? req.originalUrl.substr(1) : GBServer.globals.minInstances[0].botId; // TODO: Get only bot.
const min: GBMinInstance = GBServer.globals.minInstances.filter(p => p.instance.botId === botId)[0];
const path = Path.join(process.env.PWD, 'work', `${min.instance.botId}.gbai`, `${min.instance.botId}.gbui`, 'index.html');
if (req.originalUrl && prerender && exclude) {
const { html, status } = await ssr(
req.protocol + '://' + req.get('host') + req.originalUrl,
applyOptions.useCache,
applyOptions.cacheRefreshRate
);
return res.status(status).send(html);
const html = Fs.readFileSync(path, 'utf8');
res.status(200).send(html);
return true;
} else {
return next();
if (Fs.existsSync(path)) {
res.sendFile(path);
} else {
res.status(404);
res.end();
}
}
})
.catch(next);
}
return ssrOnDemand;
}
export { createBrowser, ssr, clearCache, ssrForBots };

1
src/app.ts
View file

@ -55,6 +55,7 @@ import auth from 'basic-auth';
import child_process from 'child_process';
import * as winston from 'winston-logs-display';
import { RootData } from './RootData.js';
import { GBSSR } from '../packages/core.gbapp/services/GBSSR.js';
/**
* General Bots open-core entry point.

1
tsconfig.json
View file

@ -1,5 +1,6 @@
{
"compilerOptions": {
"allowJs": true,
"downlevelIteration": true,
"baseUrl": "./",