z0rs
/
chromium
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove embedded puppeteer-core

This commit is contained in:
Sparticuz 2022-09-26 15:10:14 -04:00
parent 034e9e0d56
commit d44fbde653
17 changed files with 35 additions and 1385 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -15,7 +15,7 @@ test:
%.zip:
npm install --fund=false --package-lock=false
mkdir -p nodejs
npm install --prefix nodejs/ tar-fs@2.1.1 puppeteer-core@17.1.3 --bin-links=false --fund=false --omit=optional --omit=dev --package-lock=false --save=false
npm install --prefix nodejs/ tar-fs@2.1.1 --bin-links=false --fund=false --omit=optional --omit=dev --package-lock=false --save=false
npm pack
mkdir -p nodejs/node_modules/@sparticuz/chrome-aws-lambda/
tar --directory nodejs/node_modules/@sparticuz/chrome-aws-lambda/ --extract --file sparticuz-chrome-aws-lambda-*.tgz --strip-components=1

7
package.json
View File

@ -10,8 +10,7 @@
"types": "build/index.d.ts",
"files": [
"bin",
"build",
"typings"
"build"
],
"engines": {
"node": ">= 14"
@ -27,12 +26,8 @@
"@types/node": "^16.11.49",
"@types/tar-fs": "^2.0.1",
"clean-modules": "^2.0.6",
"puppeteer-core": "17.1.3",
"typescript": "^4.6.4"
},
"peerDependencies": {
"puppeteer-core": "17.1.3"
},
"bugs": {
"url": "https://github.com/Sparticuz/chrome-aws-lambda/issues"
},

48
source/hooks/adblock.ts
View File

@ -1,48 +0,0 @@
import { promises } from 'fs';
import { get } from 'https';
import { Page } from 'puppeteer-core';
let adblocker: any = null;
/**
* Enables ad blocking in page.
* Requires `@cliqz/adblocker-puppeteer` package.
*
* @param page - Page to hook to.
*/
export = async function (page: Page): Promise<Page> {
if (adblocker == null) {
const { fullLists, PuppeteerBlocker } = require('@cliqz/adblocker-puppeteer');
adblocker = await PuppeteerBlocker.fromLists(
(url: string) => {
return new Promise((resolve, reject) => {
return get(url, (response) => {
if (response.statusCode !== 200) {
return reject(`Unexpected status code: ${response.statusCode}.`);
}
let result = '';
response.on('data', (chunk) => {
result += chunk;
});
response.on('end', () => {
return resolve({ text: () => result });
});
});
});
},
fullLists,
{ enableCompression: false },
{
path: '/tmp/adblock.bin',
read: promises.readFile,
write: promises.writeFile,
}
);
}
return await adblocker.enableBlockingInPage(page).then(() => page);
}

16
source/hooks/agent.ts
View File

@ -1,16 +0,0 @@
import { Page } from 'puppeteer-core';
/**
* Removes `Headless` from the User Agent string, if present.
*
* @param page - Page to hook to.
*/
export = async function (page: Page): Promise<Page> {
let result = await page.browser().userAgent();
if (result.includes('Headless') === true) {
await page.setUserAgent(result.replace('Headless', ''));
}
return page;
};

189
source/hooks/chrome.ts
View File

@ -1,189 +0,0 @@
import { Page } from 'puppeteer-core';
import { Writeable } from '../../typings/chrome-aws-lambda';
/**
* Mocks the global `chrome` property to mimic headful Chrome.
*
* @param page - Page to hook to.
*/
export = async function (page: Page): Promise<Page> {
const handler = () => {
let alpha = Date.now();
let delta = Math.floor(500 * Math.random());
if ((window as any).chrome === undefined) {
Object.defineProperty(window, 'chrome', {
configurable: false,
enumerable: true,
value: {},
writable: true,
});
}
/**
* https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra-plugin-stealth/evasions/chrome.app/index.js
*/
if ((window as any).chrome.app === undefined) {
const InvocationError = (callback: string) => {
/**
* Truncates every line of the stack trace (with the exception of the first), until `search` is found.
*/
const truncateStackTrace = (error: Error, search: string) => {
const stack = error.stack.split('\n');
const index = stack.findIndex((value: string) => value.trim().startsWith(search));
if (index > 0) {
error.stack = [stack[0], ...stack.slice(index + 1)].join('\n');
}
return error;
};
return truncateStackTrace(new TypeError(`Error in invocation of app.${callback}()`), `at ${callback} (eval at <anonymous>`);
};
Object.defineProperty((window as any).chrome, 'app', {
value: {
InstallState: {
DISABLED: 'disabled',
INSTALLED: 'installed',
NOT_INSTALLED: 'not_installed',
},
RunningState: {
CANNOT_RUN: 'cannot_run',
READY_TO_RUN: 'ready_to_run',
RUNNING: 'running',
},
get isInstalled() {
return false;
},
getDetails: function getDetails(): null {
if (arguments.length > 0) {
throw InvocationError('getDetails');
}
return null;
},
getIsInstalled: function getIsInstalled() {
if (arguments.length > 0) {
throw InvocationError('getIsInstalled');
}
return false;
},
runningState: function runningState() {
if (arguments.length > 0) {
throw InvocationError('runningState');
}
return 'cannot_run';
},
},
});
}
let timing: Partial<PerformanceTiming> = {
navigationStart: alpha + 1 * delta,
domContentLoadedEventEnd: alpha + 4 * delta,
responseStart: alpha + 2 * delta,
loadEventEnd: alpha + 5 * delta,
};
if (window.performance?.timing !== undefined) {
timing = window.performance.timing;
}
/**
* https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth/evasions/chrome.csi
*/
if ((window as any).chrome.csi === undefined) {
Object.defineProperty((window as any).chrome, 'csi', {
value: function csi() {
return {
startE: timing.navigationStart,
onloadT: timing.domContentLoadedEventEnd,
pageT: Date.now() - timing.navigationStart + Math.random().toFixed(3),
tran: 15,
};
},
});
}
/**
* https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes
*/
if ((window as any).chrome.loadTimes === undefined) {
let navigation: Writeable<Partial<PerformanceNavigationTiming>> = {
nextHopProtocol: 'h2',
startTime: 3 * delta,
type: 'other' as any,
};
if (typeof window.performance?.getEntriesByType === 'function') {
let entries = {
navigation: window.performance.getEntriesByType('navigation') as PerformanceNavigationTiming[],
paint: window.performance.getEntriesByType('paint') as PerformanceNavigationTiming[],
};
if (entries.navigation.length > 0) {
navigation = entries.navigation.shift();
}
if (entries.paint.length > 0) {
navigation.startTime = entries.paint.shift().startTime;
}
}
Object.defineProperty((window as any).chrome, 'loadTimes', {
value: function loadTimes() {
return {
get commitLoadTime() {
return timing.responseStart / 1000;
},
get connectionInfo() {
return navigation.nextHopProtocol;
},
get finishDocumentLoadTime() {
return timing.domContentLoadedEventEnd / 1000;
},
get finishLoadTime() {
return timing.loadEventEnd / 1000;
},
get firstPaintAfterLoadTime() {
return 0;
},
get firstPaintTime() {
return parseFloat(((navigation.startTime + (window.performance?.timeOrigin ?? timing.navigationStart)) / 1000).toFixed(3));
},
get navigationType() {
return navigation.type;
},
get npnNegotiatedProtocol() {
return ['h2', 'hq'].includes(navigation.nextHopProtocol) ? navigation.nextHopProtocol : 'unknown';
},
get requestTime() {
return timing.navigationStart / 1000;
},
get startLoadTime() {
return timing.navigationStart / 1000;
},
get wasAlternateProtocolAvailable() {
return false;
},
get wasFetchedViaSpdy() {
return ['h2', 'hq'].includes(navigation.nextHopProtocol);
},
get wasNpnNegotiated() {
return ['h2', 'hq'].includes(navigation.nextHopProtocol);
},
};
},
});
};
}
await page.evaluate(handler);
await page.evaluateOnNewDocument(handler);
return page;
}

23
source/hooks/languages.ts
View File

@ -1,23 +0,0 @@
import { Page } from 'puppeteer-core';
/**
* Emulates `en-US` language.
*
* @param page - Page to hook to.
*/
export = async function (page: Page): Promise<Page> {
const handler = () => {
Object.defineProperty(Object.getPrototypeOf(navigator), 'language', {
get: () => 'en-US',
});
Object.defineProperty(Object.getPrototypeOf(navigator), 'languages', {
get: () => ['en-US', 'en'],
});
};
await page.evaluate(handler);
await page.evaluateOnNewDocument(handler);
return page;
}

28
source/hooks/permissions.ts
View File

@ -1,28 +0,0 @@
import { Page } from 'puppeteer-core';
/**
* Emulates `denied` state for all permission queries.
*
* @param page - Page to hook to.
*/
export = async function (page: Page): Promise<Page> {
const handler = () => {
let query = window.navigator.permissions.query;
(Permissions as any).prototype.query = function (parameters: PermissionDescriptor) {
if (parameters?.name?.length > 0) {
return Promise.resolve({
onchange: null,
state: 'denied',
});
}
return query(parameters);
};
};
await page.evaluate(handler);
await page.evaluateOnNewDocument(handler);
return page;
}

10
source/hooks/timezone.ts
View File

@ -1,10 +0,0 @@
import { Page } from 'puppeteer-core';
/**
* Emulates UTC timezone.
*
* @param page - Page to hook to.
*/
export = function (page: Page): Promise<Page> {
return page.emulateTimezone('UTC').then(() => page);
}

19
source/hooks/webdriver.ts
View File

@ -1,19 +0,0 @@
import { Page } from 'puppeteer-core';
/**
* Removes global `webdriver` property to mimic headful Chrome.
*
* @param page - Page to hook to.
*/
export = async function (page: Page): Promise<Page> {
const handler = () => {
Object.defineProperty(Object.getPrototypeOf(navigator), 'webdriver', {
get: () => false,
});
};
await page.evaluate(handler);
await page.evaluateOnNewDocument(handler);
return page;
}

47
source/hooks/window.ts
View File

@ -1,47 +0,0 @@
import { Page } from 'puppeteer-core';
/**
* Patches window outer dimentions to mimic headful Chrome.
*
* @param page - Page to hook to.
*/
export = async function (page: Page): Promise<Page> {
const handler = () => {
if (window.outerWidth === 0) {
Object.defineProperty(window, 'outerWidth', {
get: () => screen.availWidth,
});
}
if (window.outerHeight === 0) {
Object.defineProperty(window, 'outerHeight', {
get: () => screen.availHeight,
});
}
if (window.screenX === 0) {
Object.defineProperty(window, 'screenX', {
get: () => screen.width - screen.availWidth,
});
Object.defineProperty(window, 'screenLeft', {
get: () => screenX,
});
}
if (window.screenY === 0) {
Object.defineProperty(window, 'screenY', {
get: () => screen.height - screen.availHeight,
});
Object.defineProperty(window, 'screenTop', {
get: () => screenY,
});
}
};
await page.evaluate(handler);
await page.evaluateOnNewDocument(handler);
return page;
}

55
source/index.ts
View File

@ -1,12 +1,42 @@
/// <reference path="../typings/chrome-aws-lambda.d.ts" />
import { access, createWriteStream, existsSync, mkdirSync, readdirSync, symlink, unlinkSync } from 'fs';
import { IncomingMessage } from 'http';
import LambdaFS from './lambdafs';
import { join } from 'path';
import { PuppeteerNode, Viewport } from 'puppeteer-core';
import { URL } from 'url';
/** Viewport taken from https://github.com/puppeteer/puppeteer/blob/main/docs/api/puppeteer.viewport.md */
interface Viewport {
/**
* The page width in pixels.
*/
width: number;
/**
* The page height in pixels.
*/
height: number;
/**
* Specify device scale factor.
* See {@link https://developer.mozilla.org/en-US/docs/Web/API/Window/devicePixelRatio | devicePixelRatio} for more info.
* @defaultValue 1
*/
deviceScaleFactor?: number;
/**
* Whether the `meta viewport` tag is taken into account.
* @defaultValue false
*/
isMobile?: boolean;
/**
* Specifies if the viewport is in landscape mode.
* @defaultValue false
*/
isLandscape?: boolean;
/**
* Specify if the viewport supports touch events.
* @defaultValue false
*/
hasTouch?: boolean;
}
if (/^AWS_Lambda_nodejs(?:10|12|14|16)[.]x$/.test(process.env.AWS_EXECUTION_ENV) === true) {
if (process.env.FONTCONFIG_PATH === undefined) {
process.env.FONTCONFIG_PATH = '/tmp/aws';
@ -195,25 +225,6 @@ class Chromium {
return environments.some((key) => process.env[key] !== undefined);
}
/**
* Overloads puppeteer with useful methods and returns the resolved package.
*/
static get puppeteer(): PuppeteerNode {
for (const overload of ['Browser', 'BrowserContext', 'ElementHandle', 'Frame', 'Page']) {
require(`${__dirname}/puppeteer/lib/${overload}`);
}
try {
return require('puppeteer');
} catch (error: any) {
if (error.code !== 'MODULE_NOT_FOUND') {
throw error;
}
return require('puppeteer-core');
}
}
}
export = Chromium;

43
source/puppeteer/lib/Browser.ts
View File

@ -1,43 +0,0 @@
import { Browser, Page } from 'puppeteer-core';
import { Hook, Prototype } from '../../../typings/chrome-aws-lambda';
let Super: Prototype<Browser> = null;
try {
Super = require('puppeteer/lib/cjs/puppeteer/common/Browser.js').Browser;
} catch (error) {
Super = require('puppeteer-core/lib/cjs/puppeteer/common/Browser.js').Browser;
}
Super.prototype.defaultPage = async function (...hooks: Hook[]) {
let page: Page = null;
let pages: Page[] = await this.pages();
if (pages.length === 0) {
pages = [await this.newPage()];
}
page = pages.shift();
if (hooks != null && Array.isArray(hooks) === true) {
for (let hook of hooks) {
page = await hook(page);
}
}
return page;
};
let newPage: any = Super.prototype.newPage;
Super.prototype.newPage = async function (...hooks: Hook[]) {
let page: Page = await newPage.apply(this, arguments);
if (hooks != null && Array.isArray(hooks) === true) {
for (let hook of hooks) {
page = await hook(page);
}
}
return page;
};

43
source/puppeteer/lib/BrowserContext.ts
View File

@ -1,43 +0,0 @@
import { BrowserContext, Page } from 'puppeteer-core';
import { Hook, Prototype } from '../../../typings/chrome-aws-lambda';
let Super: Prototype<BrowserContext> = null;
try {
Super = require('puppeteer/lib/cjs/puppeteer/common/Browser.js').BrowserContext;
} catch (error) {
Super = require('puppeteer-core/lib/cjs/puppeteer/common/Browser.js').BrowserContext;
}
Super.prototype.defaultPage = async function (...hooks: Hook[]) {
let page: Page = null;
let pages: Page[] = await this.pages();
if (pages.length === 0) {
pages = [await this.newPage()];
}
page = pages.shift();
if (hooks != null && Array.isArray(hooks) === true) {
for (let hook of hooks) {
page = await hook(page);
}
}
return page;
};
let newPage: any = Super.prototype.newPage;
Super.prototype.newPage = async function (...hooks: Hook[]) {
let page: Page = await newPage.apply(this, arguments);
if (hooks != null && Array.isArray(hooks) === true) {
for (let hook of hooks) {
page = await hook(page);
}
}
return page;
};

557
source/puppeteer/lib/ElementHandle.ts
View File

@ -1,557 +0,0 @@
import { ElementHandle, EvaluateFunc, HTTPRequest, HTTPResponse, Page, WaitForOptions, WaitTimeoutOptions } from 'puppeteer-core';
import { Prototype } from '../../../typings/chrome-aws-lambda';
let Super: Prototype<ElementHandle> = null;
try {
Super = require('puppeteer/lib/cjs/puppeteer/common/ElementHandle.js').ElementHandle;
} catch (error) {
Super = require('puppeteer-core/lib/cjs/puppeteer/common/ElementHandle.js').ElementHandle;
}
Super.prototype.clear = function () {
return this.click({ clickCount: 3 }).then(() => this.press('Backspace'));
};
Super.prototype.clickAndWaitForNavigation = function (options?: WaitForOptions) {
options = options ?? {
waitUntil: [
'load',
],
};
let promises: [Promise<HTTPResponse>, Promise<void>] = [
((this as any)._page as Page).waitForNavigation(options),
this.click(),
];
return Promise.all(promises).then((value) => value.shift() as HTTPResponse);
};
Super.prototype.clickAndWaitForRequest = function (predicate: string | RegExp | ((request: HTTPRequest) => boolean | Promise<boolean>), options?: WaitTimeoutOptions) {
let callback = (request: HTTPRequest) => {
let url = request.url();
if (typeof predicate === 'string' && predicate.includes('*') === true) {
predicate = new RegExp(predicate.replace(/[-\/\\^$+?.()|[\]{}]/g, '\\$&').replace(/[*]+/g, '.*?'), 'g');
}
if (predicate instanceof RegExp) {
return predicate.test(url);
}
return predicate === url;
};
let promises: [Promise<HTTPRequest>, Promise<void>] = [
((this as any)._page as Page).waitForRequest((typeof predicate === 'function') ? predicate : callback, options),
this.click(),
];
return Promise.all(promises).then((value) => value.shift() as HTTPRequest);
};
Super.prototype.clickAndWaitForResponse = function (predicate: string | RegExp | ((request: HTTPResponse) => boolean | Promise<boolean>), options?: WaitTimeoutOptions) {
let callback = (request: HTTPResponse) => {
let url = request.url();
if (typeof predicate === 'string' && predicate.includes('*') === true) {
predicate = new RegExp(predicate.replace(/[-\/\\^$+?.()|[\]{}]/g, '\\$&').replace(/[*]+/g, '.*?'), 'g');
}
if (predicate instanceof RegExp) {
return predicate.test(url);
}
return predicate === url;
};
let promises: [Promise<HTTPResponse>, Promise<void>] = [
((this as any)._page as Page).waitForResponse((typeof predicate === 'function') ? predicate : callback, options),
this.click(),
];
return Promise.all(promises).then((value) => value.shift() as HTTPResponse);
};
Super.prototype.fillFormByLabel = function <T extends Record<string, boolean | string | string[]>>(data: T) {
let callback = (node: HTMLFormElement, data: T) => {
if (node.nodeName.toLowerCase() !== 'form') {
throw new Error('Element is not a <form> element.');
}
let result: Record<string, string[]> = {};
for (let [key, value] of Object.entries(data)) {
let selector = [
`id(string(//label[normalize-space(.) = "${key}"]/@for))`,
`//label[normalize-space(.) = "${key}"]//*[self::input or self::select or self::textarea]`,
].join(' | ');
if (result.hasOwnProperty(key) !== true) {
result[key] = [];
}
let element: Node = null;
let elements: HTMLInputElement[] = [];
let iterator = document.evaluate(selector, node, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
while ((element = iterator.iterateNext()) != null) {
elements.push(element as HTMLInputElement);
}
if (elements.length === 0) {
throw new Error(`No elements match the selector '${selector}' for '${key}'.`);
}
let type = (elements[0].getAttribute('type') || elements[0].nodeName).toLowerCase();
let values: (boolean | string)[] = (Array.isArray(value) === true) ? value as (boolean | string)[] : [value] as (boolean | string)[];
if (type === 'file') {
throw new Error(`Input element of type 'file' is not supported.`);
}
for (let element of elements) {
try {
element.focus();
element.dispatchEvent(new Event('focus'));
} catch (error) {
}
if (type === 'select') {
element.value = undefined;
for (let index of ['value', 'label'] as ['value', 'label']) {
if (result[key].length > 0) {
break;
}
for (let option of Array.from((element as unknown as HTMLSelectElement).options)) {
option.selected = values.includes(option[index]);
if (option.selected === true) {
result[key].push(option.value);
if (element.multiple !== true) {
break;
}
}
}
}
} else if (type === 'checkbox' || type === 'radio') {
element.checked = (value === true) || values.includes(element.value);
if (element.checked === true) {
result[key].push(element.value);
}
} else if (typeof value === 'string') {
if (element.isContentEditable === true) {
result[key].push(element.textContent = value);
} else {
result[key].push(element.value = value);
}
}
for (let trigger of ['input', 'change']) {
element.dispatchEvent(new Event(trigger, { 'bubbles': true }));
}
try {
element.blur();
element.dispatchEvent(new Event('blur'));
} catch (error) {
}
if (type === 'checkbox' || type === 'radio') {
break;
}
}
}
return result;
};
return this.evaluate(callback as unknown as EvaluateFunc<[ElementHandle<Element>, T]>, data) as any;
};
Super.prototype.fillFormByName = function <T extends Record<string, boolean | string | string[]>>(data: T) {
let callback = (node: HTMLFormElement, data: T, heuristic: 'css' | 'label' | 'name' | 'xpath' = 'css') => {
if (node.nodeName.toLowerCase() !== 'form') {
throw new Error('Element is not a <form> element.');
}
let result: Record<string, string[]> = {};
for (let [key, value] of Object.entries(data)) {
let selector = `[name="${key}"]`;
if (result.hasOwnProperty(key) !== true) {
result[key] = [];
}
let elements: HTMLInputElement[] = Array.from(node.querySelectorAll(selector));
if (elements.length === 0) {
throw new Error(`No elements match the selector '${selector}' for '${key}'.`);
}
let type = (elements[0].getAttribute('type') || elements[0].nodeName).toLowerCase();
let values: (boolean | string)[] = (Array.isArray(value) === true) ? value as (boolean | string)[] : [value] as (boolean | string)[];
if (type === 'file') {
throw new Error(`Input element of type 'file' is not supported.`);
}
for (let element of elements) {
try {
element.focus();
element.dispatchEvent(new Event('focus'));
} catch (error) {
}
if (type === 'select') {
element.value = undefined;
for (let index of ['value', 'label'] as ['value', 'label']) {
if (result[key].length > 0) {
break;
}
for (let option of Array.from((element as unknown as HTMLSelectElement).options)) {
option.selected = values.includes(option[index]);
if (option.selected === true) {
result[key].push(option.value);
if (element.multiple !== true) {
break;
}
}
}
}
} else if (type === 'checkbox' || type === 'radio') {
element.checked = (value === true) || values.includes(element.value);
if (element.checked === true) {
result[key].push(element.value);
}
} else if (typeof value === 'string') {
if (element.isContentEditable === true) {
result[key].push(element.textContent = value);
} else {
result[key].push(element.value = value);
}
}
for (let trigger of ['input', 'change']) {
element.dispatchEvent(new Event(trigger, { 'bubbles': true }));
}
try {
element.blur();
element.dispatchEvent(new Event('blur'));
} catch (error) {
}
if (type === 'checkbox' || type === 'radio') {
break;
}
}
}
return result;
};
return this.evaluate(callback as unknown as EvaluateFunc<[ElementHandle<Element>, T]>, data) as any;
};
Super.prototype.fillFormBySelector = function <T extends Record<string, boolean | string | string[]>>(data: T) {
let callback = (node: HTMLFormElement, data: T, heuristic: 'css' | 'label' | 'name' | 'xpath' = 'css') => {
if (node.nodeName.toLowerCase() !== 'form') {
throw new Error('Element is not a <form> element.');
}
let result: Record<string, string[]> = {};
for (let [key, value] of Object.entries(data)) {
let selector = key;
if (result.hasOwnProperty(key) !== true) {
result[key] = [];
}
let elements: HTMLInputElement[] = Array.from(node.querySelectorAll(selector));
if (elements.length === 0) {
throw new Error(`No elements match the selector '${selector}' for '${key}'.`);
}
let type = (elements[0].getAttribute('type') || elements[0].nodeName).toLowerCase();
let values: (boolean | string)[] = (Array.isArray(value) === true) ? value as (boolean | string)[] : [value] as (boolean | string)[];
if (type === 'file') {
throw new Error(`Input element of type 'file' is not supported.`);
}
for (let element of elements) {
try {
element.focus();
element.dispatchEvent(new Event('focus'));
} catch (error) {
}
if (type === 'select') {
element.value = undefined;
for (let index of ['value', 'label'] as ['value', 'label']) {
if (result[key].length > 0) {
break;
}
for (let option of Array.from((element as unknown as HTMLSelectElement).options)) {
option.selected = values.includes(option[index]);
if (option.selected === true) {
result[key].push(option.value);
if (element.multiple !== true) {
break;
}
}
}
}
} else if (type === 'checkbox' || type === 'radio') {
element.checked = (value === true) || values.includes(element.value);
if (element.checked === true) {
result[key].push(element.value);
}
} else if (typeof value === 'string') {
if (element.isContentEditable === true) {
result[key].push(element.textContent = value);
} else {
result[key].push(element.value = value);
}
}
for (let trigger of ['input', 'change']) {
element.dispatchEvent(new Event(trigger, { 'bubbles': true }));
}
try {
element.blur();
element.dispatchEvent(new Event('blur'));
} catch (error) {
}
if (type === 'checkbox' || type === 'radio') {
break;
}
}
}
return result;
};
return this.evaluate(callback as unknown as EvaluateFunc<[ElementHandle<Element>, T]>, data) as any;
};
Super.prototype.fillFormByXPath = function <T extends Record<string, boolean | string | string[]>>(data: T) {
let callback = (node: HTMLFormElement, data: T) => {
if (node.nodeName.toLowerCase() !== 'form') {
throw new Error('Element is not a <form> element.');
}
let result: Record<string, string[]> = {};
for (let [key, value] of Object.entries(data)) {
let selector = key;
if (result.hasOwnProperty(key) !== true) {
result[key] = [];
}
let element: Node = null;
let elements: HTMLInputElement[] = [];
let iterator = document.evaluate(selector, node, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
while ((element = iterator.iterateNext()) != null) {
elements.push(element as HTMLInputElement);
}
if (elements.length === 0) {
throw new Error(`No elements match the selector '${selector}' for '${key}'.`);
}
let type = (elements[0].getAttribute('type') || elements[0].nodeName).toLowerCase();
let values: (boolean | string)[] = (Array.isArray(value) === true) ? value as (boolean | string)[] : [value] as (boolean | string)[];
if (type === 'file') {
throw new Error(`Input element of type 'file' is not supported.`);
}
for (let element of elements) {
try {
element.focus();
element.dispatchEvent(new Event('focus'));
} catch (error) {
}
if (type === 'select') {
element.value = undefined;
for (let index of ['value', 'label'] as ['value', 'label']) {
if (result[key].length > 0) {
break;
}
for (let option of Array.from((element as unknown as HTMLSelectElement).options)) {
option.selected = values.includes(option[index]);
if (option.selected === true) {
result[key].push(option.value);
if (element.multiple !== true) {
break;
}
}
}
}
} else if (type === 'checkbox' || type === 'radio') {
element.checked = (value === true) || values.includes(element.value);
if (element.checked === true) {
result[key].push(element.value);
}
} else if (typeof value === 'string') {
if (element.isContentEditable === true) {
result[key].push(element.textContent = value);
} else {
result[key].push(element.value = value);
}
}
for (let trigger of ['input', 'change']) {
element.dispatchEvent(new Event(trigger, { 'bubbles': true }));
}
try {
element.blur();
element.dispatchEvent(new Event('blur'));
} catch (error) {
}
if (type === 'checkbox' || type === 'radio') {
break;
}
}
}
return result;
};
return this.evaluate(callback as unknown as EvaluateFunc<[ElementHandle<Element>, T]>, data) as any;
};
Super.prototype.getInnerHTML = function () {
return this.evaluate((node: Element) => {
return (node as HTMLElement).innerHTML;
});
};
Super.prototype.getInnerText = function () {
return this.evaluate((node: Element) => {
return (node as HTMLElement).innerText;
});
};
Super.prototype.number = function (decimal: string = '.', property: any) {
let callback = (node: any, decimal: string, property: any) => {
let data = (node[property] as unknown) as string;
if (typeof data === 'string') {
decimal = decimal ?? '.';
if (typeof decimal === 'string') {
decimal = decimal.replace(/[.]/g, '\\$&');
}
let matches = data.match(/((?:[-+]|\b)[0-9]+(?:[ ,.'`´]*[0-9]+)*)\b/g);
if (matches != null) {
return matches.map((value) => parseFloat(value.replace(new RegExp(`[^-+0-9${decimal}]+`, 'g'), '').replace(decimal, '.')));
}
}
return null;
};
return this.evaluate(callback, decimal, property as any);
};
Super.prototype.selectByLabel = function (...values: string[]) {
for (let value of values) {
console.assert(typeof value === 'string', `Values must be strings. Found value '${value}' of type '${typeof value}'.`);
}
let callback = (node: HTMLSelectElement, values: string[]) => {
if (node.nodeName.toLowerCase() !== 'select') {
throw new Error('Element is not a <select> element.');
}
node.value = undefined;
let result = [];
let options = Array.from(node.options);
for (let option of options) {
option.selected = values.includes(option.label);
if (option.selected === true) {
result.push(option.value);
if (node.multiple !== true) {
break;
}
}
}
for (let trigger of ['input', 'change']) {
node.dispatchEvent(new Event(trigger, { bubbles: true }));
}
return result;
};
return this.evaluate(callback as any, values);
};
Super.prototype.string = function (property: any) {
let callback = (node: any, property: any) => {
let data = (node[property] as unknown) as string;
if (typeof data === 'string') {
let patterns = {
' ': /[\u00A0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u202F\u205F\u3000]/g,
'-': /[\u2013\u2014]/g,
'...': /[\u2026]/g,
'': /[\u200B\uFEFF]/g,
'"': /[\u201C\u201D]/g,
'<': /[\u00AB\u2039]/g,
'>': /[\u00BB\u203A]/g,
'|': /[\u007C\u00A6\u01C0\u2223\u2758]/g,
"'": /[\u2018\u2019\u201A\u201B\u2032]/g,
};
for (let [key, value] of Object.entries(patterns)) {
data = data.replace(value, key);
}
return data.replace(/[\s]+/g, ' ').trim();
}
return null;
};
return this.evaluate(callback, property as any);
};

148
source/puppeteer/lib/Frame.ts
View File

@ -1,148 +0,0 @@
import { Frame, HTTPRequest, HTTPResponse, WaitForOptions, WaitTimeoutOptions } from 'puppeteer-core';
import { KeysOfType, Prototype } from '../../../typings/chrome-aws-lambda';
let Super: Prototype<Frame> = null;
try {
Super = require('puppeteer/lib/cjs/puppeteer/common/Frame.js').Frame;
} catch (error) {
Super = require('puppeteer-core/lib/cjs/puppeteer/common/Frame.js').Frame;
}
Super.prototype.clear = function (selector: string) {
return this.$(selector).then((element) => element?.clear());
};
Super.prototype.clickAndWaitForNavigation = function (selector: string, options?: WaitForOptions) {
options = options ?? {
waitUntil: [
'load',
],
};
let promises: [Promise<HTTPResponse>, Promise<void>] = [
this.waitForNavigation(options),
this.waitForSelector(selector, { timeout: options.timeout }).then((element) => element.click()),
];
return Promise.all(promises).then((value) => value.shift() as HTTPResponse);
};
Super.prototype.clickAndWaitForRequest = function (selector: string, predicate: string | RegExp | ((request: HTTPRequest) => boolean | Promise<boolean>), options?: WaitTimeoutOptions) {
let callback = (request: HTTPRequest) => {
let url = request.url();
if (typeof predicate === 'string' && predicate.includes('*') === true) {
predicate = new RegExp(predicate.replace(/[-\/\\^$+?.()|[\]{}]/g, '\\$&').replace(/[*]+/g, '.*?'), 'g');
}
if (predicate instanceof RegExp) {
return predicate.test(url);
}
return predicate === url;
};
let promises: [Promise<HTTPRequest>, Promise<void>] = [
this.page().waitForRequest((typeof predicate === 'function') ? predicate : callback, options),
this.click(selector),
];
return Promise.all(promises).then((value) => value.shift() as HTTPRequest);
};
Super.prototype.clickAndWaitForResponse = function (selector: string, predicate: string | RegExp | ((request: HTTPResponse) => boolean | Promise<boolean>), options?: WaitTimeoutOptions) {
let callback = (request: HTTPResponse) => {
let url = request.url();
if (typeof predicate === 'string' && predicate.includes('*') === true) {
predicate = new RegExp(predicate.replace(/[-\/\\^$+?.()|[\]{}]/g, '\\$&').replace(/[*]+/g, '.*?'), 'g');
}
if (predicate instanceof RegExp) {
return predicate.test(url);
}
return predicate === url;
};
let promises: [Promise<HTTPResponse>, Promise<void>] = [
this.page().waitForResponse((typeof predicate === 'function') ? predicate : callback, options),
this.click(selector),
];
return Promise.all(promises).then((value) => value.shift() as HTTPResponse);
};
Super.prototype.count = function (selector: string) {
let callback = (selector: string) => {
return document.querySelectorAll(selector).length;
};
return this.evaluate(callback, selector);
};
Super.prototype.exists = function (selector: string) {
let callback = (selector: string) => {
return document.querySelector(selector) !== null;
};
return this.evaluate(callback, selector);
};
Super.prototype.fillFormByLabel = function <T extends Record<string, boolean | string | string[]>>(selector: string, data: T) {
return this.$(selector).then((element) => element?.fillFormByLabel(data) ?? null);
};
Super.prototype.fillFormByName = function <T extends Record<string, boolean | string | string[]>>(selector: string, data: T) {
return this.$(selector).then((element) => element?.fillFormByName(data) ?? null);
};
Super.prototype.fillFormBySelector = function <T extends Record<string, boolean | string | string[]>>(selector: string, data: T) {
return this.$(selector).then((element) => element?.fillFormBySelector(data) ?? null);
};
Super.prototype.fillFormByXPath = function <T extends Record<string, boolean | string | string[]>>(selector: string, data: T) {
return this.$(selector).then((element) => element?.fillFormByXPath(data) ?? null);
};
Super.prototype.number = function <T = HTMLElement>(selector: string, decimal: string = '.', property: KeysOfType<T, string> = 'textContent' as any) {
return this.$(selector).then((element) => element?.number(decimal, property) ?? null);
};
Super.prototype.selectByLabel = function (selector: string, ...values: string[]) {
return this.$(selector).then((element) => element?.selectByLabel(...values) ?? null);
};
Super.prototype.string = function <T = HTMLElement>(selector: string, property: KeysOfType<T, string> = 'textContent' as any) {
return this.$(selector).then((element) => element?.string(property) ?? null);
};
Super.prototype.waitForText = function (predicate: string, options?: WaitTimeoutOptions) {
if (predicate.includes(`"`) !== true) {
predicate = `"${predicate}"`;
} else if (predicate.includes(`'`) !== true) {
predicate = `'${predicate}'`;
} else {
throw new Error('Predicate cannot include both single and double quotes.');
}
return this.waitForXPath(`//*[contains(concat(' ', normalize-space(text()), ' '), ${predicate})]`, {
...options,
visible: true,
});
};
Super.prototype.waitUntilVisible = function (selector: string, options?: WaitTimeoutOptions) {
return this.waitForSelector(selector, {
...options,
visible: true,
});
};
Super.prototype.waitWhileVisible = function (selector: string, options?: WaitTimeoutOptions) {
return this.waitForSelector(selector, {
...options,
hidden: true,
});
};

184
source/puppeteer/lib/Page.ts
View File

@ -1,184 +0,0 @@
import { CDPSession, HTTPRequest, HTTPResponse, Page, TracingOptions, WaitForOptions, WaitTimeoutOptions } from 'puppeteer-core';
import { KeysOfType, Prototype } from '../../../typings/chrome-aws-lambda';
let Super: Prototype<Page> = null;
try {
Super = require('puppeteer/lib/cjs/puppeteer/common/Page.js').Page;
} catch (error) {
Super = require('puppeteer-core/lib/cjs/puppeteer/common/Page.js').Page;
}
Super.prototype.block = function (predicates: string[]) {
return ((this as any)._client as CDPSession).send('Network.setBlockedURLs', {
urls: predicates
});
};
Super.prototype.clear = function (selector: string) {
return this.mainFrame().clear(selector);
};
Super.prototype.clickAndWaitForNavigation = function (selector: string, options?: WaitForOptions) {
return this.mainFrame().clickAndWaitForNavigation(selector, options);
};
Super.prototype.clickAndWaitForRequest = function (selector: string, predicate: string | RegExp | ((request: HTTPRequest) => boolean | Promise<boolean>), options?: WaitTimeoutOptions) {
return this.mainFrame().clickAndWaitForRequest(selector, predicate as any, options);
};
Super.prototype.clickAndWaitForResponse = function (selector: string, predicate: string | RegExp | ((request: HTTPResponse) => boolean | Promise<boolean>), options?: WaitTimeoutOptions) {
return this.mainFrame().clickAndWaitForResponse(selector, predicate as any, options);
};
Super.prototype.count = function (selector: string) {
return this.mainFrame().count(selector);
};
Super.prototype.exists = function (selector: string) {
return this.mainFrame().exists(selector);
};
Super.prototype.fillFormByLabel = function <T extends Record<string, boolean | string | string[]>>(selector: string, data: T) {
return this.mainFrame().fillFormByLabel(selector, data);
};
Super.prototype.fillFormByName = function <T extends Record<string, boolean | string | string[]>>(selector: string, data: T) {
return this.mainFrame().fillFormByName(selector, data);
};
Super.prototype.fillFormBySelector = function <T extends Record<string, boolean | string | string[]>>(selector: string, data: T) {
return this.mainFrame().fillFormBySelector(selector, data);
};
Super.prototype.fillFormByXPath = function <T extends Record<string, boolean | string | string[]>>(selector: string, data: T) {
return this.mainFrame().fillFormByXPath(selector, data);
};
Super.prototype.number = function <T = HTMLElement>(selector: string, decimal: string = '.', property: KeysOfType<T, string> = 'textContent' as any) {
return this.mainFrame().number(selector, decimal, property);
};
Super.prototype.selectByLabel = function (selector: string, ...values: string[]) {
return this.mainFrame().selectByLabel(selector, ...values);
};
Super.prototype.string = function <T = HTMLElement>(selector: string, property: KeysOfType<T, string> = 'textContent' as any) {
return this.mainFrame().string(selector, property);
};
Super.prototype.waitForInflightRequests = function (requests: number = 0, alpha: number = 500, omega: number = 500, options?: WaitTimeoutOptions) {
let result: Record<string, Function> = {
reject: null,
resolve: null,
};
let timeout: NodeJS.Timeout;
let timeoutAlpha: NodeJS.Timeout;
let timeoutOmega: NodeJS.Timeout;
if (options == null) {
options = {
timeout: (this as any)._timeoutSettings.navigationTimeout(),
};
}
let inflight = 0;
const check = () => {
if (inflight <= Math.max(0, requests)) {
if (timeoutOmega !== undefined) {
clearTimeout(timeoutOmega);
}
timeoutOmega = setTimeout(onTimeoutOmega, omega);
}
};
const clear = () => {
if (timeout !== undefined) {
clearTimeout(timeout);
}
if (timeoutAlpha !== undefined) {
clearTimeout(timeoutAlpha);
}
if (timeoutOmega !== undefined) {
clearTimeout(timeoutOmega);
}
this.off('request', onRequestStarted);
this.off('requestfailed', onRequestSettled);
this.off('requestfinished', onRequestSettled);
};
function onRequestStarted() {
if (timeoutAlpha !== undefined) {
clearTimeout(timeoutAlpha);
}
if (timeoutOmega !== undefined) {
clearTimeout(timeoutOmega);
}
++inflight;
}
function onRequestSettled() {
if (inflight > 0) {
--inflight;
}
check();
}
function onTimeout() {
clear(); return result.reject(new Error(`Navigation timeout of ${options.timeout} ms exceeded.`));
}
function onTimeoutAlpha() {
clear(); return result.resolve();
}
function onTimeoutOmega() {
clear(); return result.resolve();
}
this.on('request', onRequestStarted);
this.on('requestfailed', onRequestSettled);
this.on('requestfinished', onRequestSettled);
if (options.timeout !== 0) {
timeout = setTimeout(onTimeout, options.timeout);
}
timeoutAlpha = setTimeout(onTimeoutAlpha, alpha);
return new Promise((resolve, reject) => {
result.reject = reject;
result.resolve = resolve;
});
};
Super.prototype.waitForText = function (predicate: string, options?: WaitTimeoutOptions) {
return this.mainFrame().waitForText(predicate, options);
};
Super.prototype.waitUntilVisible = function (selector: string, options?: WaitTimeoutOptions) {
return this.mainFrame().waitUntilVisible(selector, options);
};
Super.prototype.waitWhileVisible = function (selector: string, options?: WaitTimeoutOptions) {
return this.mainFrame().waitWhileVisible(selector, options);
};
Super.prototype.withTracing = function (options: TracingOptions, callback: (page: Page) => Promise<any>): Promise<Buffer> {
return this.tracing.start(options).then(async () => {
if (typeof callback === 'function') {
await callback(this);
}
return await this.tracing.stop();
});
};

1
tsconfig.json
View File

@ -31,7 +31,6 @@
"strictNullChecks": false,
"target": "es2020",
"traceResolution": false,
"typeRoots": ["node_modules/@types", "typings"],
"types": ["node"]
},
"exclude": ["build", "node_modules", "tmp"],