Skip to content

Commit

Permalink
Do not continue taking screenshots if element cannot be screenshot
Browse files Browse the repository at this point in the history
  • Loading branch information
@younglim
younglim committed Jun 20, 2024
1 parent a2e37a6 commit 5be8e93
Show file tree
Hide file tree
Showing 2 changed files with 106 additions and 87 deletions.
178 changes: 95 additions & 83 deletions src/crawlers/custom/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,67 +19,66 @@ export const log = str => {
export const screenshotFullPage = async (page, screenshotsDir:string, screenshotIdx) => {
const imgName = `PHScan-screenshot${screenshotIdx}.png`;
const imgPath = path.join(screenshotsDir, imgName);

const fullPageSize = await page.evaluate(() => ({
width: Math.max(
document.body.scrollWidth,
document.documentElement.scrollWidth,
document.body.offsetWidth,
document.documentElement.offsetWidth,
document.body.clientWidth,
document.documentElement.clientWidth,
),
height: Math.max(
document.body.scrollHeight,
document.documentElement.scrollHeight,
document.body.offsetHeight,
document.documentElement.offsetHeight,
document.body.clientHeight,
document.documentElement.clientHeight,
),
}));

const originalSize = page.viewportSize();

const usesInfiniteScroll = async () => {
const prevHeight = await page.evaluate(() => document.body.scrollHeight);

await page.evaluate(() => {
window.scrollTo(0, document.body.scrollHeight);
});
try {
const fullPageSize = await page.evaluate(() => ({
width: Math.max(
document.body.scrollWidth,
document.documentElement.scrollWidth,
document.body.offsetWidth,
document.documentElement.offsetWidth,
document.body.clientWidth,
document.documentElement.clientWidth,
),
height: Math.max(
document.body.scrollHeight,
document.documentElement.scrollHeight,
document.body.offsetHeight,
document.documentElement.offsetHeight,
document.body.clientHeight,
document.documentElement.clientHeight,
),
}));

const usesInfiniteScroll = async () => {
const prevHeight = await page.evaluate(() => document.body.scrollHeight);

await page.evaluate(() => {
window.scrollTo(0, document.body.scrollHeight);
});

const isLoadMoreContent = async () =>
new Promise(resolve => {
setTimeout(async () => {
await page.waitForLoadState('domcontentloaded');
const isLoadMoreContent = async () =>
new Promise(resolve => {
setTimeout(async () => {
await page.waitForLoadState('domcontentloaded');

const newHeight = await page.evaluate(
// eslint-disable-next-line no-shadow
() => document.body.scrollHeight,
);
const result = newHeight > prevHeight;
const newHeight = await page.evaluate(
// eslint-disable-next-line no-shadow
() => document.body.scrollHeight,
);
const result = newHeight > prevHeight;

resolve(result);
}, 2500);
});
resolve(result);
}, 2500);
});

const result = await isLoadMoreContent();
return result;
};
const result = await isLoadMoreContent();
return result;
};

await usesInfiniteScroll();
await usesInfiniteScroll();

// scroll back to top of page for screenshot
await page.evaluate(() => {
window.scrollTo(0, 0);
});
// scroll back to top of page for screenshot
await page.evaluate(() => {
window.scrollTo(0, 0);
});

consoleLogger.info(`Screenshot page at: ${urlWithoutAuth(page.url())}`);
silentLogger.info(`Screenshot page at: ${urlWithoutAuth(page.url())}`);
consoleLogger.info(`Screenshot page at: ${urlWithoutAuth(page.url())}`);
silentLogger.info(`Screenshot page at: ${urlWithoutAuth(page.url())}`);

try {
await page.screenshot({
timeout: 10000,
timeout: 5000,
path: imgPath,
clip: {
x: 0,
Expand All @@ -95,10 +94,10 @@ export const screenshotFullPage = async (page, screenshotsDir:string, screenshot

} catch (e) {
consoleLogger.error('Unable to take screenshot');
// Do not return screenshot path if screenshot fails
return "";
}

if (originalSize) await page.setViewportSize(originalSize);

return `screenshots/${imgName}`; // relative path from reports folder
};

Expand Down Expand Up @@ -130,7 +129,8 @@ export const runAxeScan = async (
export const processPage = async (page, processPageParams) => {
// make sure to update processPageParams' scannedIdx
processPageParams.scannedIdx += 1;
const {

let {
scannedIdx,
blacklistedPatterns,
includeScreenshots,
Expand All @@ -139,15 +139,14 @@ export const processPage = async (page, processPageParams) => {
urlsCrawled,
randomToken,
} = processPageParams;

try {
await page.waitForLoadState('networkidle', { timeout: 10000 });
await page.waitForLoadState('domcontentloaded');
await page.waitForLoadState('domcontentloaded', { timeout: 10000 });
} catch (e) {
consoleLogger.info('Unable to detect networkidle');
silentLogger.info('Unable to detect networkidle');
consoleLogger.info('Unable to detect page load state');
}

log(`Scan - processPage: ${page.url()}`);
consoleLogger.info(`Attempting to scan: ${page.url()}`);

const pageUrl = page.url();

Expand All @@ -170,33 +169,46 @@ export const processPage = async (page, processPageParams) => {
// return;
// }

const initialScrollPos = await page.evaluate(() => ({
x: window.scrollX,
y: window.scrollY,
}));
try {
const initialScrollPos = await page.evaluate(() => ({
x: window.scrollX,
y: window.scrollY,
}));

const pageImagePath = await screenshotFullPage(page, intermediateScreenshotsPath, scannedIdx);


const pageImagePath = await screenshotFullPage(page, intermediateScreenshotsPath, scannedIdx);
// TODO: This is a temporary fix to not take element screenshots on pages when errors out at full page screenshot
if (pageImagePath === "") {
includeScreenshots = false;
}

guiInfoLog(guiInfoStatusTypes.SCANNED, {
numScanned: urlsCrawled.scanned.length,
urlScanned: pageUrl,
});
await runAxeScan(
page,
includeScreenshots,
randomToken,
{
pageIndex: scannedIdx,
pageImagePath,
},
dataset,
urlsCrawled,
);

await runAxeScan(
page,
includeScreenshots,
randomToken,
{
pageIndex: scannedIdx,
pageImagePath,
},
dataset,
urlsCrawled,
);
guiInfoLog(guiInfoStatusTypes.SCANNED, {
numScanned: urlsCrawled.scanned.length,
urlScanned: pageUrl,
});

await page.evaluate(pos => {
window.scrollTo(pos.x, pos.y);
}, initialScrollPos);

} catch (e) {
consoleLogger.error(`Error in scanning page: ${pageUrl}`);
}


await page.evaluate(pos => {
window.scrollTo(pos.x, pos.y);
}, initialScrollPos);
};

export const MENU_POSITION = {
Expand All @@ -205,7 +217,6 @@ export const MENU_POSITION = {
};

export const updateMenu = async (page, urlsCrawled) => {
await page.waitForLoadState('domcontentloaded');
log(`Overlay menu: updating: ${page.url()}`);
await page.evaluate(
vars => {
Expand All @@ -219,7 +230,8 @@ export const updateMenu = async (page, urlsCrawled) => {
},
{ urlsCrawled },
);
log(`Overlay menu: updating: success`);

consoleLogger.info(`Overlay menu updated`);
};

export const addOverlayMenu = async (page, urlsCrawled, menuPos) => {
Expand Down
15 changes: 11 additions & 4 deletions src/screenshotFunc/htmlScreenshotFunc.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// import { JSDOM } from "jsdom";
import { silentLogger } from '../logs.js';
import { consoleLogger, silentLogger } from '../logs.js';
import { createHash } from 'crypto';
import fs from 'fs';
import path from 'path';
Expand All @@ -22,15 +22,22 @@ export const takeScreenshotForHTMLElements = async (violations, page, randomToke
const locator = page.locator(selector);
const locators = await locator.all();
for (const currLocator of locators) {
await currLocator.scrollIntoViewIfNeeded({ timeout: locatorTimeout });
await currLocator.scrollIntoViewIfNeeded({ timeout: locatorTimeout });
const isVisible = await currLocator.isVisible();

if (isVisible) {
const buffer = await currLocator.screenshot({ timeout: locatorTimeout });
const screenshotPath = getScreenshotPath(buffer, randomToken);
node.screenshotPath = screenshotPath;
screenshotCount++;
break; // Stop looping after finding the first visible locator
} else {
consoleLogger.info(`Element at ${currLocator} is not visible`);
}

break; // Stop looping after finding the first visible locator
}
} catch (e) {
silentLogger.info(e);
consoleLogger.info(`Unable to take element screenshot at ${selector}`);
}
}
newViolationNodes.push(node);
Expand Down

0 comments on commit 5be8e93

Please sign in to comment.