Move ignores/skips to independent files

mapbox · Jul 12, 2017 · 83c7fc2 · 83c7fc2
1 parent eaed8fe
commit 83c7fc2
Show file tree

Hide file tree

Showing 71 changed files with 899 additions and 738 deletions.
diff --git a/test/ignores.json b/test/ignores.json
@@ -0,0 +1,18 @@
+{
+  "query-tests/regressions/mapbox-gl-js#4494": "https://github.com/mapbox/mapbox-gl-js/issues/2716",
+  "query-tests/symbol-features-in/tilted-outside": "https://github.com/mapbox/mapbox-gl-js/issues/4945",
+  "query-tests/symbol/panned-after-insert": "https://github.com/mapbox/mapbox-gl-js/issues/3346",
+  "query-tests/symbol/rotated-after-insert": "https://github.com/mapbox/mapbox-gl-js/issues/3346",
+  "render-tests/fill-extrusion-pattern/@2x": "https://github.com/mapbox/mapbox-gl-js/issues/3327",
+  "render-tests/fill-extrusion-pattern/function-2": "https://github.com/mapbox/mapbox-gl-js/issues/3327",
+  "render-tests/fill-extrusion-pattern/function": "https://github.com/mapbox/mapbox-gl-js/issues/3327",
+  "render-tests/fill-extrusion-pattern/literal": "https://github.com/mapbox/mapbox-gl-js/issues/3327",
+  "render-tests/fill-extrusion-pattern/missing": "https://github.com/mapbox/mapbox-gl-js/issues/3327",
+  "render-tests/fill-extrusion-pattern/opacity": "https://github.com/mapbox/mapbox-gl-js/issues/3327",
+  "render-tests/fill-outline-color/fill": "https://github.com/mapbox/mapbox-gl-js/issues/4601",
+  "render-tests/geojson/inline-linestring-fill": "current behavior is arbitrary",
+  "render-tests/raster-loading/missing": "https://github.com/mapbox/mapbox-gl-js/issues/4257",
+  "render-tests/regressions/mapbox-gl-js#3682": "skip - true",
+  "render-tests/runtime-styling/image-update-icon": "skip - https://github.com/mapbox/mapbox-gl-js/issues/4804",
+  "render-tests/runtime-styling/image-update-pattern": "skip - https://github.com/mapbox/mapbox-gl-js/issues/4804"
+}
diff --git a/test/integration/lib/harness.js b/test/integration/lib/harness.js
@@ -13,6 +13,7 @@ module.exports = function (directory, implementation, options, run) {
     const server = require('./server')();
 
     const tests = options.tests || [];
+    const ignores = options.ignores || {};
 
     function shouldRunTest(group, test) {
         if (tests.length === 0)
@@ -51,29 +52,30 @@ module.exports = function (directory, implementation, options, run) {
                 return;
             }
 
+            if (implementation === 'native' && process.env.BUILDTYPE === 'Release' && group === 'debug') {
+                console.log(colors.gray(`* skipped ${group} ${test}`));
+                return;
+            }
+
+            const id = `${path.basename(directory)}/${group}/${test}`;
+            const ignored = ignores[id];
+            if (/^skip/.test(ignored)) {
+                console.log(colors.gray(`* skipped ${group} ${test} (${ignored})`));
+                return;
+            }
+
             const style = require(path.join(directory, group, test, 'style.json'));
 
             server.localizeURLs(style);
 
             const params = Object.assign({
-                group: group,
-                test: test,
+                group,
+                test,
                 width: 512,
                 height: 512,
                 pixelRatio: 1,
                 allowed: 0.00015
-            }, style.metadata && style.metadata.test);
-
-            if (implementation === 'native' && process.env.BUILDTYPE === 'Release' && params.group === 'debug') {
-                console.log(colors.gray(`* skipped ${params.group} ${params.test}`));
-                return;
-            }
-
-            const skipped = params.skipped && params.skipped[implementation];
-            if (skipped) {
-                console.log(colors.gray(`* skipped ${params.group} ${params.test} (${skipped})`));
-                return;
-            }
+            }, style.metadata && style.metadata.test, {ignored});
 
             if ('diff' in params) {
                 if (typeof params.diff === 'number') {
@@ -83,20 +85,18 @@ module.exports = function (directory, implementation, options, run) {
                 }
             }
 
-            params.ignored = params.ignored && implementation in params.ignored;
-
             q.defer((callback) => {
                 run(style, params, (err) => {
                     if (err) return callback(err);
 
                     if (params.ignored && !params.ok) {
                         params.color = '#9E9E9E';
                         params.status = 'ignored failed';
-                        console.log(colors.white(`* ignore ${params.group} ${params.test}`));
+                        console.log(colors.white(`* ignore ${params.group} ${params.test} (${params.ignored})`));
                     } else if (params.ignored) {
                         params.color = '#E8A408';
                         params.status = 'ignored passed';
-                        console.log(colors.yellow(`* ignore ${params.group} ${params.test}`));
+                        console.log(colors.yellow(`* ignore ${params.group} ${params.test} (${params.ignored})`));
                     } else if (!params.ok) {
                         params.color = 'red';
                         params.status = 'failed';

diff --git a/test/integration/lib/render.js b/test/integration/lib/render.js
@@ -53,6 +53,12 @@ function compare(path1, path2, diffPath, callback) {
  * @param {Array<string>} [options.tests] - array of test names to run; tests not in the
  * array will be skipped. Test names can be the name of a group, or the name of a group and the name
  * of an individual test, separated by a slash.
+ * @param {Object<string>} [options.ignores] - map of test names to disable. A key is the relative
+ * path to a test directory, e.g. `"render-tests/background-color/default"`. A value is a string
+ * that by convention links to an issue that explains why the test is currently disabled. By default,
+ * disabled tests will be run, but not fail the test run if the result does not match the expected
+ * result. If the value begins with "skip", the test will not be run at all -- use this for tests
+ * that would crash the test harness entirely if they were run.
  * @param {renderFn} render - a function that performs the rendering
  * @returns {undefined} terminates the process when testing is complete
  */

diff --git a/test/integration/query-tests/geometry/multilinestring/style.json b/test/integration/query-tests/geometry/multilinestring/style.json
@@ -2,9 +2,6 @@
   "version": 8,
   "metadata": {
     "test": {
-      "ignored": {
-        "native": "needs investigation"
-      },
       "width": 512,
       "height": 512,
       "debug": true,

diff --git a/test/integration/query-tests/geometry/multipolygon/style.json b/test/integration/query-tests/geometry/multipolygon/style.json
@@ -2,9 +2,6 @@
   "version": 8,
   "metadata": {
     "test": {
-      "ignored": {
-        "native": "needs investigation"
-      },
       "width": 512,
       "height": 512,
       "debug": true,

diff --git a/test/integration/query-tests/geometry/polygon/style.json b/test/integration/query-tests/geometry/polygon/style.json
@@ -2,9 +2,6 @@
   "version": 8,
   "metadata": {
     "test": {
-      "ignored": {
-        "native": "needs investigation"
-      },
       "width": 512,
       "height": 512,
       "debug": true,

diff --git a/test/integration/query-tests/regressions/mapbox-gl-js#3534/style.json b/test/integration/query-tests/regressions/mapbox-gl-js#3534/style.json
@@ -2,9 +2,6 @@
   "version": 8,
   "metadata": {
     "test": {
-      "ignored": {
-        "native": "https://github.com/mapbox/mapbox-gl-native/issues/8193"
-      },
       "collisionDebug": true,
       "width": 500,
       "height": 500,
@@ -13,7 +10,9 @@
         271
       ],
       "queryOptions": {
-        "layers": ["counties-symbol"]
+        "layers": [
+          "counties-symbol"
+        ]
       }
     }
   },
@@ -60,4 +59,4 @@
       "interactive": true
     }
   ]
-}
+}
diff --git a/test/integration/query-tests/regressions/mapbox-gl-js#4417/style.json b/test/integration/query-tests/regressions/mapbox-gl-js#4417/style.json
@@ -7,18 +7,18 @@
       "queryGeometry": [
         32,
         16
-      ],
-      "ignored": {
-        "native": "https://github.com/mapbox/mapbox-gl-native/issues/8007"
-      }
+      ]
     }
   },
   "sources": {
     "dummy": {
       "type": "geojson",
       "data": {
         "type": "Point",
-        "coordinates": [0, 0]
+        "coordinates": [
+          0,
+          0
+        ]
       }
     },
     "mapbox": {
@@ -74,4 +74,4 @@
       "source": "dummy"
     }
   ]
-}
+}
diff --git a/test/integration/query-tests/regressions/mapbox-gl-js#4494/style.json b/test/integration/query-tests/regressions/mapbox-gl-js#4494/style.json
@@ -8,10 +8,7 @@
         32,
         32
       ],
-      "debug": true,
-      "ignored": {
-        "js": "https://github.com/mapbox/mapbox-gl-js/issues/2716"
-      }
+      "debug": true
     }
   },
   "zoom": 0,
@@ -39,4 +36,4 @@
       "source": "geojson"
     }
   ]
-}
+}
diff --git a/test/integration/query-tests/symbol-features-in/pitched-screen/style.json b/test/integration/query-tests/symbol-features-in/pitched-screen/style.json
@@ -2,22 +2,19 @@
   "version": 8,
   "metadata": {
     "test": {
-      "ignored": {
-        "native": "https://github.com/mapbox/mapbox-gl-native/issues/6817"
-      },
       "collisionDebug": true,
       "debug": true,
       "width": 1000,
       "height": 1000,
       "queryGeometry": [
-          [
-              100,
-              85
-          ],
-          [
-              850,
-              400
-          ]
+        [
+          100,
+          85
+        ],
+        [
+          850,
+          400
+        ]
       ]
     }
   },
@@ -54,4 +51,4 @@
       "interactive": true
     }
   ]
-}
+}
diff --git a/test/integration/query-tests/symbol-features-in/tilted-inside/style.json b/test/integration/query-tests/symbol-features-in/tilted-inside/style.json
@@ -2,9 +2,6 @@
   "version": 8,
   "metadata": {
     "test": {
-      "ignored": {
-        "native": "https://github.com/mapbox/mapbox-gl-native/issues/5056"
-      },
       "collisionDebug": true,
       "width": 500,
       "height": 500,

diff --git a/test/integration/query-tests/symbol-features-in/tilted-outside/style.json b/test/integration/query-tests/symbol-features-in/tilted-outside/style.json
@@ -2,10 +2,6 @@
   "version": 8,
   "metadata": {
     "test": {
-      "ignored": {
-        "native": "https://github.com/mapbox/mapbox-gl-native/issues/9435",
-        "js": "https://github.com/mapbox/mapbox-gl-js/issues/4945"
-      },
       "collisionDebug": true,
       "width": 500,
       "height": 500,

diff --git a/test/integration/query-tests/symbol/panned-after-insert/style.json b/test/integration/query-tests/symbol/panned-after-insert/style.json
@@ -5,27 +5,24 @@
       "width": 256,
       "height": 256,
       "operations": [
+        [
+          "setCenter",
           [
-              "setCenter",
-              [
-                  1,
-                  1
-              ]
+            1,
+            1
           ]
+        ]
       ],
       "queryGeometry": [
-          [
-              0,
-              0
-          ],
-          [
-              256,
-              256
-          ]
-      ],
-      "ignored": {
-        "js": "https://github.com/mapbox/mapbox-gl-js/issues/3346"
-      }
+        [
+          0,
+          0
+        ],
+        [
+          256,
+          256
+        ]
+      ]
     }
   },
   "center": [
@@ -56,4 +53,4 @@
       }
     }
   ]
-}
+}
diff --git a/test/integration/query-tests/symbol/rotated-after-insert/style.json b/test/integration/query-tests/symbol/rotated-after-insert/style.json
@@ -5,24 +5,21 @@
       "width": 256,
       "height": 256,
       "operations": [
-          [
-              "setBearing",
-              45
-          ]
+        [
+          "setBearing",
+          45
+        ]
       ],
       "queryGeometry": [
-          [
-              0,
-              0
-          ],
-          [
-              256,
-              256
-          ]
-      ],
-      "ignored": {
-        "js": "https://github.com/mapbox/mapbox-gl-js/issues/3346"
-      }
+        [
+          0,
+          0
+        ],
+        [
+          256,
+          256
+        ]
+      ]
     }
   },
   "center": [
@@ -53,4 +50,4 @@
       }
     }
   ]
-}
+}
diff --git a/test/integration/query-tests/world-wrapping/box/style.json b/test/integration/query-tests/world-wrapping/box/style.json
@@ -2,9 +2,6 @@
   "version": 8,
   "metadata": {
     "test": {
-      "skipped": {
-        "native": "needs issue"
-      },
       "width": 2000,
       "height": 500,
       "pixelRatio": 0.25,

diff --git a/test/integration/query-tests/world-wrapping/point/style.json b/test/integration/query-tests/world-wrapping/point/style.json
@@ -2,9 +2,6 @@
   "version": 8,
   "metadata": {
     "test": {
-      "skipped": {
-        "native": "needs issue"
-      },
       "debug": true,
       "width": 1000,
       "height": 250,