Merge branch 'main' into sitespeed-timeout

.github/workflows/regression-test-css.yml

@@ -49,7 +49,7 @@ jobs:
         sudo apt-get --only-upgrade install google-chrome-stable
         google-chrome --version
     - name: Setup npm packages
-      run: npm install --production
+      run: npm install --omit=dev
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |

.github/workflows/regression-test-google-lighthouse-based.yml

@@ -33,7 +33,7 @@ jobs:
       with:
         node-version: '20.x'
     - name: Setup npm packages
-      run: npm install --production
+      run: npm install --omit=dev
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |

.github/workflows/regression-test-html.yml

@@ -49,7 +49,7 @@ jobs:
         sudo apt-get --only-upgrade install google-chrome-stable
         google-chrome --version
     - name: Setup npm packages
-      run: npm install --production
+      run: npm install --omit=dev
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |

.github/workflows/regression-test-http.yml

@@ -47,7 +47,7 @@ jobs:
         sudo apt-get --only-upgrade install google-chrome-stable
         google-chrome --version
     - name: Setup npm packages
-      run: npm install --production
+      run: npm install --omit=dev
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |

.github/workflows/regression-test-pa11y.yml

@@ -34,7 +34,7 @@ jobs:
       with:
         node-version: '20.x'
     - name: Setup npm packages
-      run: npm install --production
+      run: npm install --omit=dev
     # Havn't yet been able to start Google Chrome in headless mode in pa11y since move to Chrome in Pa11y 5
     - if: ${{ matrix.os == 'ubuntu-20.04' }}
       name: RUNNING TEST - LINUX

.github/workflows/regression-test-sitespeed.yml

@@ -63,7 +63,7 @@ jobs:
         sudo apt-get --only-upgrade install google-chrome-stable
         google-chrome --version
     - name: Setup npm packages
-      run: npm install --production
+      run: npm install --omit=dev
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |

.github/workflows/regression-test-software.yml

@@ -44,7 +44,7 @@ jobs:
         sudo apt-get --only-upgrade install google-chrome-stable
         google-chrome --version
     - name: Setup npm packages
-      run: npm install --production
+      run: npm install --omit=dev
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |

.github/workflows/regression-test-tracking.yml

@@ -99,7 +99,7 @@ jobs:
       name: Setup GeckoDriver (Used for Selenium)
       run: wget -q -O - https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz | tar -xvzf -
     - name: Setup npm packages
-      run: npm install --production
+      run: npm install --omit=dev
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |

.github/workflows/regression-test-ylt.yml

@@ -39,7 +39,7 @@ jobs:
       run: sudo apt-get install libjpeg-dev libfontconfig
       shell: bash
     - name: Setup npm packages
-      run: npm install --production
+      run: npm install --omit=dev
     - if: ${{ matrix.os == 'ubuntu-latest' }}
       name: RUNNING TEST - LINUX
       run: |

.github/workflows/update-software.yml

@@ -42,7 +42,7 @@ jobs:
         sudo apt-get --only-upgrade install google-chrome-stable
         google-chrome --version
     - name: Setup npm packages
-      run: npm install --production
+      run: npm install --omit=dev
     - name: Update USER_AGENT in our defaults/settings.json
       run: python update_software.py -b
     - name: Checkout advisory-database repo

Dockerfile

@@ -1,4 +1,4 @@
-FROM sitespeedio/sitespeed.io:35.0.0
+FROM sitespeedio/sitespeed.io:35.2.0
 
 USER root
 
@@ -62,7 +62,7 @@ RUN chown --recursive sitespeedio:sitespeedio /usr/src/runner
 # Run everything after as non-privileged user.
 USER sitespeedio
 
-RUN npm install --production
+RUN npm install --omit=dev
 
 RUN python3.12 -m pip install -r requirements.txt --break-system-packages && \
     python3.12 -m pip install --upgrade pip --break-system-packages && \

defaults/settings.json

@@ -12,7 +12,7 @@
             "details": false,
             "improve-only": true
         },
-        "useragent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0",
+        "useragent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:131.0) Gecko/20100101 Firefox/131.0",
         "cache": {
             "use": false,
             "max-age": 60

docs/getting-started-local.md

@@ -13,7 +13,7 @@ it is also best/fastest when wanting to contribute with new tests, translations
 - Download and install Node.js (version 20.x)
 - Download and install Google Chrome browser
 * Download and install Mozilla Firefox
-- Install required npm packages by typing following and hit Enter: `npm install --production`
+- Install required npm packages by typing following and hit Enter: `npm install --omit=dev`
 - Validate that core functionality is working by typing following and hit Enter `python default.py -h`
 - If the output looks something like example in [options and arguments](getting-started.md#options-and-arguments) you have successfully setup the general parts of webperf-core.
 - Please look at/return to the [specific test](tests/README.md) you want to run to make sure it doesn't require more steps.

docs/tests/css.md

@@ -53,7 +53,7 @@ Read more on the [general page for github actions](../getting-started-github-act
 
 * Download and install Node.js (version 20.x)
 * Download and install Google Chrome browser
-* Install NPM packages ( `npm install --production` )
+* Install NPM packages ( `npm install --omit=dev` )
 * Set `sitespeed_use_docker = False` in your `config.py`
 
 ##### Windows Specific

docs/tests/google-lighthouse-based.md

@@ -35,7 +35,7 @@ Follow the instructions below depending on what you choose.
 Benefit of this option is that you can use it to test pre production urls like your AcceptanceTest environment.
 
 * Download and install Node.js (version 20.x)
-* Install NPM packages ( `npm install --production` )
+* Install NPM packages ( `npm install --omit=dev` )
 
 ## Read more
 

docs/tests/html.md

@@ -43,7 +43,7 @@ Read more on the [general page for github actions](../getting-started-github-act
 
 * Download and install Node.js (version 20.x)
 * Download and install Google Chrome browser
-* Install NPM packages ( `npm install --production` )
+* Install NPM packages ( `npm install --omit=dev` )
 * Set `sitespeed_use_docker = False` in your `config.py`
 
 ##### Windows Specific

docs/tests/http.md

@@ -71,7 +71,7 @@ Read more on the [general page for github actions](../getting-started-github-act
 * Download and install Node.js (version 20.x)
 * Download and install Google Chrome browser
 * Download and install Mozilla Firefox browser
-* Install NPM packages ( `npm install --production` )
+* Install NPM packages ( `npm install --omit=dev` )
 * Set `sitespeed_use_docker = False` in your `config.py`
 
 ##### Windows Specific

docs/tests/pa11y.md

@@ -44,7 +44,7 @@ Read more on the [general page for github actions](../getting-started-github-act
 * Follow [general local setup steps for this repository](../getting-started-local.md)
 * Download and install Node.js (version 20.x)
 * Download and install Google Chrome browser
-* Install NPM packages ( `npm install --production` )
+* Install NPM packages ( `npm install --omit=dev` )
 
 ## FAQ
 

docs/tests/sitespeed.md

@@ -73,7 +73,7 @@ Read more on the [general page for github actions](../getting-started-github-act
 * Install ffmpeg `sudo apt install ffmpeg`
 * Download and install Node.js (version 20.x)
 * Download and install Google Chrome browser
-* Install NPM packages ( `npm install --production` )
+* Install NPM packages ( `npm install --omit=dev` )
 * Set `sitespeed_use_docker = False` in your `config.py`
 
 (You can always see [GitHub Actions SiteSpeed](../../.github/workflows/regression-test-sitespeed.yml) for all steps required line by line)
@@ -86,7 +86,7 @@ Read more on the [general page for github actions](../getting-started-github-act
 * Download and Install ffmpeg `https://ffmpeg.org/download.html#build-windows`
 * Download and install Node.js (version 20.x)
 * Download and install Google Chrome browser
-* Install NPM packages ( `npm install --production` )
+* Install NPM packages ( `npm install --omit=dev` )
 * Set `sitespeed_use_docker = False` in your `config.py`
 
 #### Using Docker image

docs/tests/software.md

@@ -140,7 +140,7 @@ TODO: Add links to blogs and articles showing how to remove info regarding what
 
 * Download and install Node.js (version 20.x)
 * Download and install Google Chrome browser
-* Install NPM packages ( `npm install --production` )
+* Install NPM packages ( `npm install --omit=dev` )
 * Set `sitespeed_use_docker = False` in your `config.py`
 
 ##### Windows Specific

docs/tests/tracking.md

@@ -143,7 +143,7 @@ This section has not been written yet.
 * Download and install Node.js (version 20.x)
 * Download and install Google Chrome browser
 * Download and install Mozilla Firefox
-* Install NPM packages ( `npm install --production` )
+* Install NPM packages ( `npm install --omit=dev` )
 * Set `sitespeed_use_docker = False` in your `config.py`
 
 ##### Windows Specific

docs/tests/yellowlab.md

@@ -32,11 +32,11 @@ Read more on the [general page for github actions](../getting-started-github-act
 * Download and install Node.js (version 20.x)
 * Download and install node-gyp `npm install -g node-gyp`
 * Setup libjpeg and fontconfig `sudo apt-get install libjpeg-dev libfontconfig`
-* Install NPM packages ( `npm install --production` )
+* Install NPM packages ( `npm install --omit=dev` )
 
 ##### On Windows:
 * Download and install Node.js (version 20.x)
-* Install NPM packages ( `npm install --production` )
+* Install NPM packages ( `npm install --omit=dev` )
 
 ## FAQ
 

engines/sql.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
-from engines.utils import use_item
 
-def write_tests(output_filename, site_tests, input_skip, input_take, _, _2):
+def write_tests(output_filename, site_tests, _, _2):
     """
     Writes site test results to a file from a given list of site tests.
 
@@ -17,33 +16,30 @@ def write_tests(output_filename, site_tests, input_skip, input_take, _, _2):
         None
     """
     with open(output_filename, 'w', encoding='utf-8') as outfile:
-        current_index = 0
         for test in site_tests:
-            if use_item(current_index, input_skip, input_take):
-                # update testresult for all sites
-                format_str = """INSERT INTO sitetests (site_id, test_date, type_of_test,
-                check_report, check_report_sec, check_report_perf, check_report_a11y, check_report_stand,
-                json_check_data, most_recent, rating, rating_sec, rating_perf, rating_a11y, rating_stand)
-                VALUES ("{siteid}", "{testdate}", "{testtype}", "{report}", "{report_sec}",
-                  "{report_perf}", "{report_a11y}", "{report_stand}", "{json}", "{recent}",
-                  "{rating}", "{rating_sec}", "{rating_perf}", "{rating_a11y}",
-                  "{rating_stand}");\n"""
-                sql_command = format_str.format(
-                    siteid=test["site_id"],
-                    testdate=test["date"],
-                    testtype=test["type_of_test"],
-                    report=test["report"],
-                    report_sec=test["report_sec"],
-                    report_perf=test["report_perf"],
-                    report_a11y=test["report_a11y"],
-                    report_stand=test["report_stand"],
-                    json=test["data"],
-                    recent=1,
-                    rating=test["rating"],
-                    rating_sec=test["rating_sec"],
-                    rating_perf=test["rating_perf"],
-                    rating_a11y=test["rating_a11y"],
-                    rating_stand=test["rating_stand"]
+            # update testresult for all sites
+            format_str = """INSERT INTO sitetests (site_id, test_date, type_of_test,
+            check_report, check_report_sec, check_report_perf, check_report_a11y, check_report_stand,
+            json_check_data, most_recent, rating, rating_sec, rating_perf, rating_a11y, rating_stand)
+            VALUES ("{siteid}", "{testdate}", "{testtype}", "{report}", "{report_sec}",
+              "{report_perf}", "{report_a11y}", "{report_stand}", "{json}", "{recent}",
+              "{rating}", "{rating_sec}", "{rating_perf}", "{rating_a11y}",
+              "{rating_stand}");\n"""
+            sql_command = format_str.format(
+                siteid=test["site_id"],
+                testdate=test["date"],
+                testtype=test["type_of_test"],
+                report=test["report"],
+                report_sec=test["report_sec"],
+                report_perf=test["report_perf"],
+                report_a11y=test["report_a11y"],
+                report_stand=test["report_stand"],
+                json=test["data"],
+                recent=1,
+                rating=test["rating"],
+                rating_sec=test["rating_sec"],
+                rating_perf=test["rating_perf"],
+                rating_a11y=test["rating_a11y"],
+                rating_stand=test["rating_stand"]
                 )
-            current_index += 1
             outfile.write(sql_command)

helpers/csp_helper.py

@@ -18,6 +18,7 @@
     'manifest-src','object-src','script-src-attr',
     'script-src-elem','style-src-attr','style-src-elem','worker-src']
 CSP_POLICIES_SELF_ALLOWED = [
+    'style-src-elem','style-src',
     'font-src','connect-src','frame-src','img-src','media-src',
     'frame-ancestors','base-uri','form-action','child-src','manifest-src']
 CSP_POLICIES_FALLBACK_SRC = [
@@ -1412,6 +1413,7 @@ def append_csp_data(req_url, req_domain, res, org_domain, result):
         bool: True if there is a match in the CSP findings, False otherwise.
     """
     csp_findings_match = False
+    # TODO: Remove text empty check when sitespeed has fixed https://github.com/sitespeedio/sitespeed.io/issues/4295
     if 'content' in res and 'text' in res['content'] and res['content']['text'] != '':
         if 'mimeType' in res['content'] and 'text/html' in res['content']['mimeType']:
             csp_findings_match = csp_findings_match or append_csp_data_for_html(