extend unfurler to dynamically render search crawler requests

unfurler/src/concurrency/ReusableSSRPage.ts

@@ -0,0 +1,65 @@
+import * as puppeteer from 'puppeteer';
+import { timeoutExecute } from 'puppeteer-cluster/dist/util';
+import logger from '../logger';
+import config from '../config';
+import ReusablePage from './ReusablePage';
+const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
+
+const mockImageBuffer = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII=", 'base64');
+
+interface RepairablePage extends puppeteer.Page {
+  repairRequested?: boolean;
+  language?: string | null;
+  createdAt?: number;
+  free?: boolean;
+  index?: number;
+}
+
+export default class ReusableSSRPage extends ReusablePage {
+
+  public constructor(options: puppeteer.LaunchOptions, puppeteer: any) {
+    super(options, puppeteer);
+  }
+
+  public async close() {
+    await (this.browser as puppeteer.Browser).close();
+  }
+
+  protected async initPage(): Promise<RepairablePage> {
+    const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage;
+    page.language = null;
+    page.createdAt = Date.now();
+    const defaultUrl = mempoolHost + '/about';
+
+    page.on('pageerror', (err) => {
+      console.log(err);
+      // page.repairRequested = true;
+    });
+    await page.setRequestInterception(true);
+      page.on('request', req => {
+        if (req.isInterceptResolutionHandled()) {
+          return req.continue();
+        }
+        if (req.resourceType() === 'image') {
+          return req.respond({
+            contentType: 'image/png',
+            headers: {"Access-Control-Allow-Origin": "*"},
+            body: mockImageBuffer
+          });
+        } else if (!['document', 'script', 'xhr', 'fetch'].includes(req.resourceType())) {
+          return req.abort();
+        } else {
+          return req.continue();
+        }
+      });
+    try {
+      await page.goto(defaultUrl, { waitUntil: "networkidle0" });
+      await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 });
+    } catch (e) {
+      logger.err(`failed to load frontend during ssr page initialization: ` + (e instanceof Error ? e.message : `${e}`));
+      page.repairRequested = true;
+    }
+    page.free = true;
+    return page
+  }
+}