GeneralBots/botserver
2
1
Fork 0
Code Issues Pull requests 1 Projects Releases 323 Packages Wiki Activity Actions

fix(basic.gblib): #282 Fix SSR for Bots 3.0.

Browse source
This commit is contained in:
rodrigorodriguez 2023-02-18 16:48:40 -03:00
parent bc85f714ca
commit 7f3bd7d8fe
7 changed files with 253 additions and 283 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
packages/basic.gblib/services/ChartServices.ts
View file

@ -32,7 +32,7 @@
'use strict'; 'use strict';
import { createBrowser } from '../../core.gbapp/services/GBSSR.js'; import { GBSSR }from '../../core.gbapp/services/GBSSR.js';
export class ChartServices { export class ChartServices {
/** /**
@ -41,7 +41,7 @@ export class ChartServices {
* @param {string} path screenshot image full path with file name * @param {string} path screenshot image full path with file name
*/ */
public static async screenshot (args, path) { public static async screenshot (args, path) {
const browser = await createBrowser(null); const browser = await GBSSR.createBrowser(null);
const page = await browser.newPage(); const page = await browser.newPage();
// load billboard.js assets from CDN. // load billboard.js assets from CDN.

4
packages/basic.gblib/services/SystemKeywords.ts
View file

@ -39,7 +39,7 @@ import { DialogKeywords } from './DialogKeywords.js';
import { GBServer } from '../../../src/app.js'; import { GBServer } from '../../../src/app.js';
import { GBVMService } from './GBVMService.js'; import { GBVMService } from './GBVMService.js';
import Fs from 'fs'; import Fs from 'fs';
import { createBrowser } from '../../core.gbapp/services/GBSSR.js'; import { GBSSR }from '../../core.gbapp/services/GBSSR.js';
import urlJoin from 'url-join'; import urlJoin from 'url-join';
import Excel from 'exceljs'; import Excel from 'exceljs';
import { TwitterApi } from 'twitter-api-v2'; import { TwitterApi } from 'twitter-api-v2';
@ -257,7 +257,7 @@ export class SystemKeywords {
const { min, user } = await DialogKeywords.getProcessInfo(pid); const { min, user } = await DialogKeywords.getProcessInfo(pid);
const gbaiName = `${min.botId}.gbai`; const gbaiName = `${min.botId}.gbai`;
const browser = await createBrowser(null); const browser = await GBSSR.createBrowser(null);
const page = await browser.newPage(); const page = await browser.newPage();
// Includes the associated CSS related to current theme. // Includes the associated CSS related to current theme.

39
packages/basic.gblib/services/WebAutomationServices.ts
View file

@ -32,19 +32,19 @@
'use strict'; 'use strict';
import { GBLog, GBMinInstance } from 'botlib';
import { GBServer } from '../../../src/app.js';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
import { createBrowser } from '../../core.gbapp/services/GBSSR.js';
import { GuaribasUser } from '../../security.gbapp/models/index.js';
import { DialogKeywords } from './DialogKeywords.js';
import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
import urlJoin from 'url-join'; import urlJoin from 'url-join';
import Fs from 'fs'; import Fs from 'fs';
import Path from 'path'; import Path from 'path';
import url from 'url'; import url from 'url';
import { Mutex, Semaphore, withTimeout } from 'async-mutex';
import { GBLog, GBMinInstance } from 'botlib';
import { GBServer } from '../../../src/app.js';
import { GBAdminService } from '../../admin.gbapp/services/GBAdminService.js';
import { GBSSR }from '../../core.gbapp/services/GBSSR.js';
import { GuaribasUser } from '../../security.gbapp/models/index.js';
import { DialogKeywords } from './DialogKeywords.js';
import { GBDeployer } from '../../core.gbapp/services/GBDeployer.js';
import { Mutex } from 'async-mutex';
import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js'; import { GBLogEx } from '../../core.gbapp/services/GBLogEx.js';
/** /**
@ -172,16 +172,17 @@ export class WebAutomationServices {
let browser; let browser;
if (!page) { if (!page) {
browser = await createBrowser(null); browser = await GBSSR.createBrowser(null);
page = (await browser.pages())[0]; page = (await browser.pages())[0];
if (username || password) { if (username || password) {
await page.authenticate({ pid, username: username, password: password }); await page.authenticate({ pid, username: username, password: password });
} }
} }
// There is no session yet, // There is no session yet.
if (!session && sessionKind === 'AS') { if (!session && sessionKind === 'AS') {
// A new web session is being created. // A new web session is being created.
handle = WebAutomationServices.cyrb53(this.min.botId + url); handle = WebAutomationServices.cyrb53(this.min.botId + url);
@ -459,4 +460,20 @@ export class WebAutomationServices {
return file; return file;
} }
private async recursiveFindInFrames (inputFrame, selector) {
const frames = inputFrame.childFrames();
const results = await Promise.all(
frames.map(async frame => {
const el = await frame.$(selector);
if (el) return el;
if (frame.childFrames().length > 0) {
return await this.recursiveFindInFrames(frame, selector);
}
return null;
})
);
return results.find(Boolean);
}
} }

16
packages/core.gbapp/services/GBMinService.ts
View file

@ -87,6 +87,7 @@ import { GoogleChatDirectLine } from '../../google-chat.gblib/services/GoogleCha
import { SystemKeywords } from '../../basic.gblib/services/SystemKeywords.js'; import { SystemKeywords } from '../../basic.gblib/services/SystemKeywords.js';
import * as nlp from 'node-nlp'; import * as nlp from 'node-nlp';
import Path from 'path'; import Path from 'path';
import { GBSSR } from './GBSSR.js';
/** /**
* Minimal service layer for a bot and encapsulation of BOT Framework calls. * Minimal service layer for a bot and encapsulation of BOT Framework calls.
@ -236,6 +237,7 @@ export class GBMinService {
const url = `/api/messages/${botId}`; const url = `/api/messages/${botId}`;
removeRoute(GBServer.globals.server, url); removeRoute(GBServer.globals.server, url);
const uiUrl = `/${botId}`; const uiUrl = `/${botId}`;
removeRoute(GBServer.globals.server, uiUrl); removeRoute(GBServer.globals.server, uiUrl);
@ -295,6 +297,10 @@ export class GBMinService {
if (!Fs.existsSync(dir)) { if (!Fs.existsSync(dir)) {
mkdirp.sync(dir); mkdirp.sync(dir);
} }
dir = `work/${min.botId}.gbai/${min.botId}.gbui`;
if (!Fs.existsSync(dir)) {
mkdirp.sync(dir);
}
// Loads Named Entity data for this bot. // Loads Named Entity data for this bot.
@ -370,15 +376,17 @@ export class GBMinService {
if (process.env.DISABLE_WEB !== 'true') { if (process.env.DISABLE_WEB !== 'true') {
const uiUrl = `/${instance.botId}`; const uiUrl = `/${instance.botId}`;
let staticHandler = express.static(urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build'));
GBServer.globals.server.get(uiUrl, async (req, res, next)=> {
await GBSSR.ssrFilter(req, res, staticHandler as any);
});
const uiUrlAlt = `/${instance.activationCode}`; const uiUrlAlt = `/${instance.activationCode}`;
GBServer.globals.server.use(
uiUrl,
express.static(urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build'))
);
GBServer.globals.server.use( GBServer.globals.server.use(
uiUrlAlt, uiUrlAlt,
express.static(urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build')) express.static(urlJoin(GBDeployer.deployFolder, GBMinService.uiPackage, 'build'))
); );
const domain = min.core.getParam(min.instance, 'Domain', null); const domain = min.core.getParam(min.instance, 'Domain', null);
if (domain) { if (domain) {
GBServer.globals.server.use( GBServer.globals.server.use(

467
packages/core.gbapp/services/GBSSR.ts
View file

@ -36,297 +36,240 @@
'use strict'; 'use strict';
import Path from 'path';
import {createRequire} from "module";
const require = createRequire(import.meta.url);
const puppeteer = require('puppeteer-extra');
const hidden = require('puppeteer-extra-plugin-stealth')
// require executablePath from puppeteer
const {executablePath} = require('puppeteer')
import Fs from 'fs'; import Fs from 'fs';
import { NextFunction, Request, Response } from 'express'; import { NextFunction, Request, Response } from 'express';
import urljoin from 'url-join'; import urljoin from 'url-join';
import { GBMinInstance } from 'botlib';
import { GBServer } from '../../../src/app.js';
import { GBLogEx } from './GBLogEx.js';
import { createRequire } from 'module';
const require = createRequire(import.meta.url);
const puppeteer = require('puppeteer-extra');
const hidden = require('puppeteer-extra-plugin-stealth');
const { executablePath } = require('puppeteer');
// https://hackernoon.com/tips-and-tricks-for-web-scraping-with-puppeteer-ed391a63d952 export class GBSSR {
// Dont download all resources, we just need the HTML // https://hackernoon.com/tips-and-tricks-for-web-scraping-with-puppeteer-ed391a63d952
// Also, this is huge performance/response time boost // Dont download all resources, we just need the HTML
const blockedResourceTypes = ['image', 'media', 'font', 'texttrack', 'object', 'beacon', 'csp_report', 'imageset']; // Also, this is huge performance/response time boost
// const whitelist = ["document", "script", "xhr", "fetch"]; private blockedResourceTypes = ['image', 'media', 'font', 'texttrack', 'object', 'beacon', 'csp_report', 'imageset'];
const skippedResources = [
'quantserve',
'adzerk',
'doubleclick',
'adition',
'exelator',
'sharethrough',
'cdn.api.twitter',
'google-analytics',
'googletagmanager',
'google',
'fontawesome',
'facebook',
'analytics',
'optimizely',
'clicktale',
'mixpanel',
'zedo',
'clicksor',
'tiqcdn'
];
const RENDER_CACHE = new Map(); // const whitelist = ["document", "script", "xhr", "fetch"];
private skippedResources = [
async function createBrowser (profilePath): Promise<any> { 'quantserve',
let args = [ 'adzerk',
'--check-for-update-interval=2592000', 'doubleclick',
'--disable-accelerated-2d-canvas', 'adition',
'--disable-dev-shm-usage', 'exelator',
'--disable-features=site-per-process', 'sharethrough',
'--disable-gpu', 'cdn.api.twitter',
'--no-first-run', 'google-analytics',
'--no-default-browser-check' 'googletagmanager',
'google',
'fontawesome',
'facebook',
'analytics',
'optimizely',
'clicktale',
'mixpanel',
'zedo',
'clicksor',
'tiqcdn'
]; ];
if (profilePath) { public static async createBrowser(profilePath): Promise<any> {
args.push(`--user-data-dir=${profilePath}`); let args = [
'--check-for-update-interval=2592000',
'--disable-accelerated-2d-canvas',
'--disable-dev-shm-usage',
'--disable-features=site-per-process',
'--disable-gpu',
'--no-first-run',
'--no-default-browser-check'
];
const preferences = urljoin(profilePath, 'Default', 'Preferences'); if (profilePath) {
if (Fs.existsSync(preferences)) { args.push(`--user-data-dir=${profilePath}`);
const file = Fs.readFileSync(preferences, 'utf8');
const data = JSON.parse(file); const preferences = urljoin(profilePath, 'Default', 'Preferences');
data['profile']['exit_type'] = 'none'; if (Fs.existsSync(preferences)) {
Fs.writeFileSync(preferences, JSON.stringify(data)); const file = Fs.readFileSync(preferences, 'utf8');
const data = JSON.parse(file);
data['profile']['exit_type'] = 'none';
Fs.writeFileSync(preferences, JSON.stringify(data));
}
} }
puppeteer.use(hidden());
const browser = await puppeteer.launch({
args: args,
ignoreHTTPSErrors: true,
headless: false,
defaultViewport: null,
executablePath: executablePath(),
ignoreDefaultArgs: ['--enable-automation', '--enable-blink-features=IdleDetection']
});
return browser;
} }
puppeteer.use(hidden())
const browser = await puppeteer.launch({
args: args,
ignoreHTTPSErrors: true,
headless: false,
defaultViewport: null,
executablePath:executablePath(),
ignoreDefaultArgs: ['--enable-automation', '--enable-blink-features=IdleDetection']
});
return browser;
}
async function recursiveFindInFrames (inputFrame, selector) { /**
const frames = inputFrame.childFrames(); * Return the HTML of bot default.gbui.
const results = await Promise.all( */
frames.map(async frame => { public async getHTML(min: GBMinInstance) {
const el = await frame.$(selector); const url = urljoin(GBServer.globals.publicAddress, min.botId);
if (el) return el; const browser = await GBSSR.createBrowser(null);
if (frame.childFrames().length > 0) { const stylesheetContents = {};
return await recursiveFindInFrames(frame, selector);
}
return null;
})
);
return results.find(Boolean);
}
/** try {
* https://developers.google.com/web/tools/puppeteer/articles/ssr#reuseinstance const page = await browser.newPage();
* @param {string} url URL to prerender. await page.setUserAgent(
*/ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
async function ssr (url: string, useCache: boolean, cacheRefreshRate: number) { );
if (RENDER_CACHE.has(url) && useCache) { await page.setRequestInterception(true);
const cached = RENDER_CACHE.get(url); page.on('request', request => {
if (Date.now() - cached.renderedAt > cacheRefreshRate && !(cacheRefreshRate <= 0)) { const requestUrl = request.url().split('?')[0].split('#')[0];
RENDER_CACHE.delete(url); if (
} else { this.blockedResourceTypes.indexOf(request.resourceType()) !== -1 ||
return { this.skippedResources.some(resource => requestUrl.indexOf(resource) !== -1)
html: cached.html, ) {
status: 200 request.abort();
}; } else {
} request.continue();
} }
const browser = await createBrowser(null);
const stylesheetContents = {};
try {
const page = await browser.newPage();
await page.setUserAgent(
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
);
await page.setRequestInterception(true);
page.on('request', request => {
const requestUrl = request
.url()
.split('?')[0]
.split('#')[0];
if (
blockedResourceTypes.indexOf(request.resourceType()) !== -1 ||
skippedResources.some(resource => requestUrl.indexOf(resource) !== -1)
) {
request.abort();
} else {
request.continue();
}
});
page.on('response', async resp => {
const responseUrl = resp.url();
const sameOrigin = new URL(responseUrl).origin === new URL(url).origin;
const isStylesheet = resp.request().resourceType() === 'stylesheet';
if (sameOrigin && isStylesheet) {
stylesheetContents[responseUrl] = await resp.text();
}
});
const response = await page.goto(url, {
timeout: 120000,
waitUntil: 'networkidle0'
});
const sleep = ms => {
return new Promise(resolve => {
setTimeout(resolve, ms);
}); });
};
await sleep(45000);
// Inject <base> on page to relative resources load properly. page.on('response', async resp => {
await page.evaluate(url => { const responseUrl = resp.url();
const base = document.createElement('base'); const sameOrigin = new URL(responseUrl).origin === new URL(url).origin;
base.href = url; const isStylesheet = resp.request().resourceType() === 'stylesheet';
// Add to top of head, before all other resources. if (sameOrigin && isStylesheet) {
document.head.prepend(base); stylesheetContents[responseUrl] = await resp.text();
}, url); }
// Remove scripts and html imports. They've already executed.
await page.evaluate(() => {
const elements = document.querySelectorAll('script, link[rel="import"]');
elements.forEach(e => {
e.remove();
}); });
});
// Replace stylesheets in the page with their equivalent <style>. const response = await page.goto(url, {
await page.$$eval( timeout: 120000,
'link[rel="stylesheet"]', waitUntil: 'networkidle0'
(links, content) => { });
links.forEach((link: any) => {
const cssText = content[link.href]; const sleep = ms => {
if (cssText) { return new Promise(resolve => {
const style = document.createElement('style'); setTimeout(resolve, ms);
style.textContent = cssText;
link.replaceWith(style);
}
}); });
}, };
stylesheetContents await sleep(45000);
);
const html = await page.content(); // Inject <base> on page to relative resources load properly.
// Close the page we opened here (not the browser). await page.evaluate(url => {
await page.close(); const base = document.createElement('base');
if (useCache) { base.href = url;
RENDER_CACHE.set(url, { html, renderedAt: Date.now() }); // Add to top of head, before all other resources.
document.head.prepend(base);
}, url);
// Remove scripts and html imports. They've already executed.
await page.evaluate(() => {
const elements = document.querySelectorAll('script, link[rel="import"]');
elements.forEach(e => {
e.remove();
});
});
// Replace stylesheets in the page with their equivalent <style>.
await page.$$eval(
'link[rel="stylesheet"]',
(links, content) => {
links.forEach((link: any) => {
const cssText = content[link.href];
if (cssText) {
const style = document.createElement('style');
style.textContent = cssText;
link.replaceWith(style);
}
});
},
stylesheetContents
);
const html = await page.content();
// Close the page we opened here (not the browser).
await page.close();
return html;
} catch (e) {
const html = e.toString();
GBLogEx.error(min, `URL: ${url} Failed with message: ${html}`);
return html;
} finally {
await browser.close();
} }
return { html, status: response!.status() };
} catch (e) {
const html = e.toString();
console.warn({ message: `URL: ${url} Failed with message: ${html}` });
return { html, status: 500 };
} finally {
await browser.close();
} }
}
function clearCache () { public static async ssrFilter(req: Request, res: Response, next) {
RENDER_CACHE.clear(); let applyOptions = {
}
interface Options {
prerender?: Array<string>;
exclude?: Array<string>;
useCache?: boolean;
cacheRefreshRate?: number;
}
function ssrForBots (
options: Options = {
prerender: [], // Array containing the user-agents that will trigger the ssr service
exclude: [], // Array containing paths and/or extentions that will be excluded from being prerendered by the ssr service
useCache: true, // Variable that determins if we will use page caching or not
cacheRefreshRate: 86400 // Seconds of which the cache will be kept alive, pass 0 or negative value for infinite lifespan
}
) {
let applyOptions = Object.assign(
{
prerender: [], // Array containing the user-agents that will trigger the ssr service prerender: [], // Array containing the user-agents that will trigger the ssr service
exclude: [], // Array containing paths and/or extentions that will be excluded from being prerendered by the ssr service exclude: [], // Array containing paths and/or extentions that will be excluded from being prerendered by the ssr service
useCache: true, // Variable that determins if we will use page caching or not useCache: true, // Variable that determins if we will use page caching or not
cacheRefreshRate: 86400 // Seconds of which the cache will be kept alive, pass 0 or negative value for infinite lifespan cacheRefreshRate: 86400 // Seconds of which the cache will be kept alive, pass 0 or negative value for infinite lifespan
}, };
options
);
// Default user agents // Default user agents
const prerenderArray = [ const prerenderArray = [
'bot', 'bot',
'googlebot', 'googlebot',
'Chrome-Lighthouse', 'Chrome-Lighthouse',
'DuckDuckBot', 'DuckDuckBot',
'ia_archiver', 'ia_archiver',
'bingbot', 'bingbot',
'yandex', 'yandex',
'baiduspider', 'baiduspider',
'Facebot', 'Facebot',
'facebookexternalhit', 'facebookexternalhit',
'facebookexternalhit/1.1', 'facebookexternalhit/1.1',
'twitterbot', 'twitterbot',
'rogerbot', 'rogerbot',
'linkedinbot', 'linkedinbot',
'embedly', 'embedly',
'quora link preview', 'quora link preview',
'showyoubot', 'showyoubot',
'outbrain', 'outbrain',
'pinterest', 'pinterest',
'slackbot', 'slackbot',
'vkShare', 'vkShare',
'W3C_Validator' 'W3C_Validator'
]; ];
// default exclude array // default exclude array
const excludeArray = ['.xml', '.ico', '.txt', '.json']; const excludeArray = ['.xml', '.ico', '.txt', '.json'];
const userAgent: string = req.headers['user-agent'] || '';
const prerender = new RegExp([...prerenderArray, ...applyOptions.prerender].join('|').slice(0, -1), 'i').test(
userAgent
);
const exclude = !new RegExp([...excludeArray, ...applyOptions.exclude].join('|').slice(0, -1)).test(
req.originalUrl
);
function ssrOnDemand (req: Request, res: Response, next: NextFunction) { // Reads from static HTML when a bot is crawling.
Promise.resolve(() => {
const botId = req.originalUrl ? req.originalUrl.substr(1) : GBServer.globals.minInstances[0].botId; // TODO: Get only bot.
const min: GBMinInstance = GBServer.globals.minInstances.filter(p => p.instance.botId === botId)[0];
const path = Path.join(process.env.PWD, 'work', `${min.instance.botId}.gbai`, `${min.instance.botId}.gbui`, 'index.html');
if (req.originalUrl && prerender && exclude) {
const html = Fs.readFileSync(path, 'utf8');
res.status(200).send(html);
return true; return true;
}) } else {
.then(async () => { if (Fs.existsSync(path)) {
const userAgent: string = req.headers['user-agent'] || ''; res.sendFile(path);
} else {
const prerender = new RegExp([...prerenderArray, ...applyOptions.prerender].join('|').slice(0, -1), 'i').test( res.status(404);
userAgent res.end();
); }
}
const exclude = !new RegExp([...excludeArray, ...applyOptions.exclude].join('|').slice(0, -1)).test(
req.originalUrl
);
if (req.originalUrl && prerender && exclude) {
const { html, status } = await ssr(
req.protocol + '://' + req.get('host') + req.originalUrl,
applyOptions.useCache,
applyOptions.cacheRefreshRate
);
return res.status(status).send(html);
} else {
return next();
}
})
.catch(next);
} }
return ssrOnDemand;
} }
export { createBrowser, ssr, clearCache, ssrForBots };

5
src/app.ts
View file

@ -55,6 +55,7 @@ import auth from 'basic-auth';
import child_process from 'child_process'; import child_process from 'child_process';
import * as winston from 'winston-logs-display'; import * as winston from 'winston-logs-display';
import { RootData } from './RootData.js'; import { RootData } from './RootData.js';
import { GBSSR } from '../packages/core.gbapp/services/GBSSR.js';
/** /**
* General Bots open-core entry point. * General Bots open-core entry point.
@ -97,7 +98,7 @@ export class GBServer {
server.use(bodyParser.json()); server.use(bodyParser.json());
server.use(bodyParser.urlencoded({ extended: true })); server.use(bodyParser.urlencoded({ extended: true }));
process.on('unhandledRejection', (err, p) => { process.on('unhandledRejection', (err, p) => {
GBLog.error(`UNHANDLED_REJECTION(promises): ${p} ${err.toString()}`); GBLog.error(`UNHANDLED_REJECTION(promises): ${p} ${err.toString()}`);
}); });
@ -229,7 +230,7 @@ export class GBServer {
winston.default(server, loggers[1]); winston.default(server, loggers[1]);
} }
server.get('*', function(req, res){ server.get('*', function (req, res) {
GBLog.info(`HTTP 404: ${req.url}.`); GBLog.info(`HTTP 404: ${req.url}.`);
res.status(404); res.status(404);
res.end(); res.end();

1
tsconfig.json
View file

@ -1,5 +1,6 @@
{ {
"compilerOptions": { "compilerOptions": {
"allowJs": true, "allowJs": true,
"downlevelIteration": true, "downlevelIteration": true,
"baseUrl": "./", "baseUrl": "./",