From 94fcd88bf6d77c58a499bac23c727046eb4e4922 Mon Sep 17 00:00:00 2001 From: Alex Tkachev Date: Fri, 19 Jan 2024 22:29:03 +0400 Subject: [PATCH] chore: extract saveWebpage to a dedicated package --- pnpm-lock.yaml | 215 ++----------------- tests/e2e/package.json | 5 +- tests/e2e/webform-snapshots/webforms.spec.ts | 43 +--- 3 files changed, 25 insertions(+), 238 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8070d597d..0f9b0cc92 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -540,10 +540,10 @@ importers: version: 6.0.3(postcss@8.4.32) eslint-plugin-formatjs: specifier: ^4.11.3 - version: 4.11.3(eslint@8.56.0) + version: 4.11.3(eslint@7.32.0) eslint-plugin-storybook: specifier: ^0.6.15 - version: 0.6.15(eslint@8.56.0)(typescript@5.3.3) + version: 0.6.15(eslint@7.32.0)(typescript@5.3.3) eslint-plugin-tailwindcss: specifier: ^3.13.1 version: 3.13.1(tailwindcss@3.4.0) @@ -604,18 +604,15 @@ importers: tests/e2e: devDependencies: + '@amazeelabs/save-webpage': + specifier: ^1.1.1 + version: 1.1.1 '@playwright/test': specifier: ^1.40.1 version: 1.40.1 '@types/node': specifier: ^18 version: 18.19.4 - '@types/website-scraper': - specifier: ^1.2.10 - version: 1.2.10 - website-scraper: - specifier: ^5.3.1 - version: 5.3.1 tests/schema: devDependencies: @@ -951,6 +948,14 @@ packages: typescript: 5.3.3 dev: false + /@amazeelabs/save-webpage@1.1.1: + resolution: {integrity: sha512-TAVJFHbBB54njYraGJ86xuOhrrYOGb/CYOP9c2BBShG7CsvTF+6v/4XApM2HWYyfxZH5kCUBNyRj6q+VjXiFqQ==} + dependencies: + website-scraper: 5.3.1 + transitivePeerDependencies: + - supports-color + dev: true + /@amazeelabs/scalars@1.6.13(react@18.2.0)(tailwindcss@3.4.0): resolution: {integrity: sha512-XhIdqShLa0cW3TSY+dZU/xnAiRorQ4rXnAT3+A+LYnF8mGprZC4UsbyMmt79YYz6Plkgpa1qEQ0Um50bN+p8xQ==} peerDependencies: @@ -3675,16 +3680,6 @@ packages: eslint: 7.32.0 eslint-visitor-keys: 3.4.3 - /@eslint-community/eslint-utils@4.4.0(eslint@8.56.0): - resolution: {integrity: sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - peerDependencies: - eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 - dependencies: - eslint: 8.56.0 - eslint-visitor-keys: 3.4.3 - dev: true - /@eslint-community/regexpp@4.10.0: resolution: {integrity: sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==} engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} @@ -3706,28 +3701,6 @@ packages: transitivePeerDependencies: - supports-color - /@eslint/eslintrc@2.1.4: - resolution: {integrity: sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - dependencies: - ajv: 6.12.6 - debug: 4.3.4 - espree: 9.6.1 - globals: 13.24.0 - ignore: 5.3.0 - import-fresh: 3.3.0 - js-yaml: 4.1.0 - minimatch: 3.1.2 - strip-json-comments: 3.1.1 - transitivePeerDependencies: - - supports-color - dev: true - - /@eslint/js@8.56.0: - resolution: {integrity: sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - dev: true - /@fal-works/esbuild-plugin-global-externals@2.1.2: resolution: {integrity: sha512-cEee/Z+I12mZcFJshKcCqC8tuX5hG3s+d+9nZ3LabqKF1vKdF41B92pJVCBggjAGORAeOzyyDDKrZwIkLffeOQ==} dev: true @@ -4778,17 +4751,6 @@ packages: react-hook-form: 7.49.2(react@18.2.0) dev: false - /@humanwhocodes/config-array@0.11.14: - resolution: {integrity: sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==} - engines: {node: '>=10.10.0'} - dependencies: - '@humanwhocodes/object-schema': 2.0.2 - debug: 4.3.4 - minimatch: 3.1.2 - transitivePeerDependencies: - - supports-color - dev: true - /@humanwhocodes/config-array@0.5.0: resolution: {integrity: sha512-FagtKFz74XrTl7y6HCzQpwDfXP0yhxe9lHLD1UZxjvZIcbyRz8zTFF/yYNfSfzU414eDwZ1SrO0Qvtyf+wFMQg==} engines: {node: '>=10.10.0'} @@ -4800,11 +4762,6 @@ packages: transitivePeerDependencies: - supports-color - /@humanwhocodes/module-importer@1.0.1: - resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} - engines: {node: '>=12.22'} - dev: true - /@humanwhocodes/momoa@2.0.4: resolution: {integrity: sha512-RE815I4arJFtt+FVeU1Tgp9/Xvecacji8w/V6XtXsWWH/wz/eNkNbhb+ny/+PlVZjV0rxQpRSQKNKE3lcktHEA==} engines: {node: '>=10.10.0'} @@ -4814,10 +4771,6 @@ packages: resolution: {integrity: sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==} requiresBuild: true - /@humanwhocodes/object-schema@2.0.2: - resolution: {integrity: sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw==} - dev: true - /@iarna/toml@2.2.5: resolution: {integrity: sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==} dev: false @@ -9412,10 +9365,6 @@ packages: '@types/node': 18.19.4 '@types/responselike': 1.0.3 - /@types/caseless@0.12.5: - resolution: {integrity: sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==} - dev: true - /@types/chai-subset@1.3.5: resolution: {integrity: sha512-c2mPnw+xHtXDoHmdtcCXGwyLMiauiAyxWMzhGpqHC4nqI/Y5G2XhTampslK2rb59kpcuHon03UH8W6iYUzw88A==} requiresBuild: true @@ -9764,15 +9713,6 @@ packages: '@types/scheduler': 0.16.8 csstype: 3.1.3 - /@types/request@2.48.12: - resolution: {integrity: sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==} - dependencies: - '@types/caseless': 0.12.5 - '@types/node': 18.19.4 - '@types/tough-cookie': 4.0.5 - form-data: 2.5.1 - dev: true - /@types/resolve@1.20.6: resolution: {integrity: sha512-A4STmOXPhMUtHH+S6ymgE2GiBSMqf4oTvcQZMcHzokuTLVYzXTB8ttjcgxOVaAp2lGwEdzZ0J+cRbbeevQj1UQ==} dev: true @@ -9830,10 +9770,6 @@ packages: /@types/tmp@0.0.33: resolution: {integrity: sha512-gVC1InwyVrO326wbBZw+AO3u2vRXz/iRWq9jYhpG4W8LXyIgDv3ZmcLQ5Q4Gs+gFMyqx+viFoFT+l3p61QFCmQ==} - /@types/tough-cookie@4.0.5: - resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==} - dev: true - /@types/triple-beam@1.3.5: resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==} @@ -9873,12 +9809,6 @@ packages: '@types/node': 18.19.4 dev: true - /@types/website-scraper@1.2.10: - resolution: {integrity: sha512-yJR4klQv1/7PuqMmgm331re3gQLKHt3dtCYQPPDwlCdthZgE/zHhWyb3XsXCVt6IPnMnOb62I5oJIZECqfvx1A==} - dependencies: - '@types/request': 2.48.12 - dev: true - /@types/wordpress__block-editor@11.5.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-Q4aY88Q2eq3At5tLDEHeR7CFRrf+CHx8+JdMq8RMlsZFdnUKGoRVrqCg8sKqVTelljEtiBCbSisbgf4UltnOxg==} dependencies: @@ -10373,26 +10303,6 @@ packages: - supports-color - typescript - /@typescript-eslint/utils@5.62.0(eslint@8.56.0)(typescript@5.3.3): - resolution: {integrity: sha512-n8oxjeb5aIbPFEtmQxQYOLI0i9n5ySBEY/ZEHHZqKQSFnxio1rv6dthascc9dLuwrL0RC5mPCxB7vnAVGAYWAQ==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - peerDependencies: - eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 - dependencies: - '@eslint-community/eslint-utils': 4.4.0(eslint@8.56.0) - '@types/json-schema': 7.0.15 - '@types/semver': 7.5.6 - '@typescript-eslint/scope-manager': 5.62.0 - '@typescript-eslint/types': 5.62.0 - '@typescript-eslint/typescript-estree': 5.62.0(typescript@5.3.3) - eslint: 8.56.0 - eslint-scope: 5.1.1 - semver: 7.5.4 - transitivePeerDependencies: - - supports-color - - typescript - dev: true - /@typescript-eslint/utils@6.17.0(eslint@7.32.0)(typescript@5.3.3): resolution: {integrity: sha512-LofsSPjN/ITNkzV47hxas2JCsNCEnGhVvocfyOcLzT9c/tSZE7SfhS/iWtzP1lKNOEfLhRTZz6xqI8N2RzweSQ==} engines: {node: ^16.0.0 || >=18.0.0} @@ -10412,25 +10322,6 @@ packages: - typescript dev: true - /@typescript-eslint/utils@6.17.0(eslint@8.56.0)(typescript@5.3.3): - resolution: {integrity: sha512-LofsSPjN/ITNkzV47hxas2JCsNCEnGhVvocfyOcLzT9c/tSZE7SfhS/iWtzP1lKNOEfLhRTZz6xqI8N2RzweSQ==} - engines: {node: ^16.0.0 || >=18.0.0} - peerDependencies: - eslint: ^7.0.0 || ^8.0.0 - dependencies: - '@eslint-community/eslint-utils': 4.4.0(eslint@8.56.0) - '@types/json-schema': 7.0.15 - '@types/semver': 7.5.6 - '@typescript-eslint/scope-manager': 6.17.0 - '@typescript-eslint/types': 6.17.0 - '@typescript-eslint/typescript-estree': 6.17.0(typescript@5.3.3) - eslint: 8.56.0 - semver: 7.5.4 - transitivePeerDependencies: - - supports-color - - typescript - dev: true - /@typescript-eslint/visitor-keys@5.62.0: resolution: {integrity: sha512-07ny+LHRzQXepkGg6w0mFY41fVUNBrL2Roj/++7V1txKugfjm/Ci/qSND03r2RhlJhJYMcTn9AhhSSqQp0Ysyw==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -10446,10 +10337,6 @@ packages: eslint-visitor-keys: 3.4.3 dev: true - /@ungap/structured-clone@1.2.0: - resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} - dev: true - /@vercel/nft@0.23.1: resolution: {integrity: sha512-NE0xSmGWVhgHF1OIoir71XAd0W0C1UE3nzFyhpFiMr3rVhetww7NvM1kc41trBsPG37Bh+dE5FYCTMzM/gBu0w==} engines: {node: '>=14'} @@ -16414,7 +16301,7 @@ packages: lodash: 4.17.21 string-natural-compare: 3.0.1 - /eslint-plugin-formatjs@4.11.3(eslint@8.56.0): + /eslint-plugin-formatjs@4.11.3(eslint@7.32.0): resolution: {integrity: sha512-VGmDGbRZexN+rpweaXBoXkZ40mu96zk1fi1A+iVDAhxIyQr4QLZyhHWwMM1JXgGxgGCB90/buxkRl95nzSn10w==} peerDependencies: eslint: 7 || 8 @@ -16423,9 +16310,9 @@ packages: '@formatjs/ts-transformer': 3.13.9 '@types/eslint': 8.56.1 '@types/picomatch': 2.3.3 - '@typescript-eslint/utils': 6.17.0(eslint@8.56.0)(typescript@5.3.3) + '@typescript-eslint/utils': 6.17.0(eslint@7.32.0)(typescript@5.3.3) emoji-regex: 10.3.0 - eslint: 8.56.0 + eslint: 7.32.0 magic-string: 0.30.5 picomatch: 2.3.1 tslib: 2.6.2 @@ -16581,15 +16468,15 @@ packages: dependencies: eslint: 7.32.0 - /eslint-plugin-storybook@0.6.15(eslint@8.56.0)(typescript@5.3.3): + /eslint-plugin-storybook@0.6.15(eslint@7.32.0)(typescript@5.3.3): resolution: {integrity: sha512-lAGqVAJGob47Griu29KXYowI4G7KwMoJDOkEip8ujikuDLxU+oWJ1l0WL6F2oDO4QiyUFXvtDkEkISMOPzo+7w==} engines: {node: 12.x || 14.x || >= 16} peerDependencies: eslint: '>=6' dependencies: '@storybook/csf': 0.0.1 - '@typescript-eslint/utils': 5.62.0(eslint@8.56.0)(typescript@5.3.3) - eslint: 8.56.0 + '@typescript-eslint/utils': 5.62.0(eslint@7.32.0)(typescript@5.3.3) + eslint: 7.32.0 requireindex: 1.2.0 ts-dedent: 2.2.0 transitivePeerDependencies: @@ -16615,14 +16502,6 @@ packages: esrecurse: 4.3.0 estraverse: 4.3.0 - /eslint-scope@7.2.2: - resolution: {integrity: sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - dependencies: - esrecurse: 4.3.0 - estraverse: 5.3.0 - dev: true - /eslint-utils@2.1.0: resolution: {integrity: sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==} engines: {node: '>=6'} @@ -16710,53 +16589,6 @@ packages: transitivePeerDependencies: - supports-color - /eslint@8.56.0: - resolution: {integrity: sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ==} - engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - hasBin: true - dependencies: - '@eslint-community/eslint-utils': 4.4.0(eslint@8.56.0) - '@eslint-community/regexpp': 4.10.0 - '@eslint/eslintrc': 2.1.4 - '@eslint/js': 8.56.0 - '@humanwhocodes/config-array': 0.11.14 - '@humanwhocodes/module-importer': 1.0.1 - '@nodelib/fs.walk': 1.2.8 - '@ungap/structured-clone': 1.2.0 - ajv: 6.12.6 - chalk: 4.1.2 - cross-spawn: 7.0.3 - debug: 4.3.4 - doctrine: 3.0.0 - escape-string-regexp: 4.0.0 - eslint-scope: 7.2.2 - eslint-visitor-keys: 3.4.3 - espree: 9.6.1 - esquery: 1.5.0 - esutils: 2.0.3 - fast-deep-equal: 3.1.3 - file-entry-cache: 6.0.1 - find-up: 5.0.0 - glob-parent: 6.0.2 - globals: 13.24.0 - graphemer: 1.4.0 - ignore: 5.3.0 - imurmurhash: 0.1.4 - is-glob: 4.0.3 - is-path-inside: 3.0.3 - js-yaml: 4.1.0 - json-stable-stringify-without-jsonify: 1.0.1 - levn: 0.4.1 - lodash.merge: 4.6.2 - minimatch: 3.1.2 - natural-compare: 1.4.0 - optionator: 0.9.3 - strip-ansi: 6.0.1 - text-table: 0.2.0 - transitivePeerDependencies: - - supports-color - dev: true - /espree@7.3.1: resolution: {integrity: sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==} engines: {node: ^10.12.0 || >=12.0.0} @@ -17741,15 +17573,6 @@ packages: requiresBuild: true dev: true - /form-data@2.5.1: - resolution: {integrity: sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==} - engines: {node: '>= 0.12'} - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - dev: true - /form-data@4.0.0: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} engines: {node: '>= 6'} diff --git a/tests/e2e/package.json b/tests/e2e/package.json index fe3051f3a..fafd90c69 100644 --- a/tests/e2e/package.json +++ b/tests/e2e/package.json @@ -16,9 +16,8 @@ "download-browsers": "playwright install chromium" }, "devDependencies": { + "@amazeelabs/save-webpage": "^1.1.1", "@playwright/test": "^1.40.1", - "@types/node": "^18", - "@types/website-scraper": "^1.2.10", - "website-scraper": "^5.3.1" + "@types/node": "^18" } } diff --git a/tests/e2e/webform-snapshots/webforms.spec.ts b/tests/e2e/webform-snapshots/webforms.spec.ts index 27b6684bb..29d153e94 100644 --- a/tests/e2e/webform-snapshots/webforms.spec.ts +++ b/tests/e2e/webform-snapshots/webforms.spec.ts @@ -1,6 +1,6 @@ +import { saveWebpage } from '@amazeelabs/save-webpage'; import { expect, Page, test } from '@playwright/test'; import { execSync } from 'child_process'; -import scrape from 'website-scraper'; import { cmsUrl } from '../helpers/url'; @@ -23,44 +23,9 @@ test('Export webforms for styling', async ({ page }) => { }); async function savePage(page: Page, name: string) { - await scrape({ + await saveWebpage({ directory: `${baseDir}/${name}`, - urls: [page.url()], - plugins: [new ScrapePlugin(await page.content())], + url: page.url(), + content: await page.content(), }); } - -// Replaces the first request response with given content. -class ScrapePlugin { - isFirstRequest = true; - - constructor(public firstRequestResponse: string) {} - - apply( - registerAction: ( - event: 'afterResponse', - handler: (args: { response: T }) => Promise< - | T - | { - body: string; - encoding: 'utf8'; - metadata: {}; - } - >, - ) => void, - ): void { - registerAction('afterResponse', async ({ response }) => { - if (this.isFirstRequest) { - this.isFirstRequest = false; - return { - body: this.firstRequestResponse, - encoding: 'utf8', - metadata: {}, - }; - } else { - this.isFirstRequest = false; - return response; - } - }); - } -}