Enable URL rewriting for Github license requests.
* Also enables the new headless mode in Puppeteer.
* Adding a `.puppeteerrc.cjs` to make sure App Engine Standard correctly downloads the correct version of Puppeteer.
Test: curl -d '{"url": "https://github.com/google/desugar_jdk_libs/blob/master/LICENSE"}' -H 'Content-Type: application/json' -X POST 'http://localhost:8080/convert/licenses'
Fixes: b/285652313
Change-Id: I8fdc0ea21008dd4cabd5d81b28bf7bfa3ddc8079
diff --git a/development/fetchLicenses/license.ts b/development/fetchLicenses/license.ts
index 8c3579b..b51cc82 100644
--- a/development/fetchLicenses/license.ts
+++ b/development/fetchLicenses/license.ts
@@ -19,6 +19,7 @@
import { log } from './logger';
import { ContentNode } from './types';
import { PlainTextFormatter } from './plain_text_formatter';
+import { transformUrl } from './url-transforms';
const CHROME_LAUNCH_ARGS = ['--enable-dom-distiller'];
@@ -73,14 +74,23 @@
}
async function handleLicenseRequest(url: string, enableLocalDebugging: boolean = false): Promise<ContentNode[]> {
- const browser = await puppeteer.launch({ args: CHROME_LAUNCH_ARGS, devtools: enableLocalDebugging });
+ const transformed = transformUrl(url);
+ if (url !== transformed) {
+ log(`Transformed request url to ${transformed}`);
+ }
+ const browser = await puppeteer.launch({
+ args: CHROME_LAUNCH_ARGS,
+ devtools: enableLocalDebugging,
+ // https://developer.chrome.com/articles/new-headless/
+ headless: 'new'
+ });
const page = await browser.newPage();
if (enableLocalDebugging) {
page.on('console', (message) => {
log(`Puppeteer: ${message.text()}`);
});
}
- await page.goto(url, { waitUntil: 'domcontentloaded' });
+ await page.goto(transformed, { waitUntil: 'domcontentloaded' });
const content = await page.evaluate(() => {
// A map of banned nodes
const BANNED_LOCAL_NAMES: BannedNames = {
