URL: Use test data from web-platform-tests for isURL spec conformance (…

…#20537)

* URL: Use test data from web-platform-tests for isURL spec conformance

* URL: Change fetch utility to parse JSON

Early iterations included LICENSE fetching as string, but this was later changed to static file. As such, the utility is better served to include JSON parsing.

* URL: Stop fetch execution when rejecting by status code

Interestingly, rejecting (throwing) does not itself halt execution.

* URL: Update fetchJSON function description

* URL: Try using react-native-url-polyfill

* URL: Clarify default exceptions as about:blank parameter specific

* URL: Add Native-specific isURL exceptions

* URL: Move WPT license into explanatory README.md file

packages/url/package.json

@@ -24,7 +24,8 @@
 	"dependencies": {
 		"@babel/runtime": "^7.8.3",
 		"lodash": "^4.17.15",
-		"qs": "^6.5.2"
+		"qs": "^6.5.2",
+		"react-native-url-polyfill": "^1.1.2"
 	},
 	"publishConfig": {
 		"access": "public"

packages/url/scripts/download-wpt-data.js

@@ -0,0 +1,118 @@
+/**
+ * External dependencies
+ */
+const { get } = require( 'https' );
+const path = require( 'path' );
+const fs = require( 'fs' );
+
+/**
+ * Default file output destination.
+ */
+const DEFAULT_OUT_FILE = path.resolve(
+	__dirname,
+	'../src/test/fixtures/wpt-data.json'
+);
+
+/**
+ * Source test data URL.
+ */
+const DATA_URL =
+	'https://raw.githubusercontent.com/web-platform-tests/wpt/master/url/resources/urltestdata.json';
+
+/**
+ * Items to exclude from the default test data, where the test case relies on
+ * an explicit `'about:blank'` base parameter provided to the constructor. The
+ * test data as given does not otherwise allow for distinction between a null
+ * base and base of `'about:blank'`.
+ *
+ * @type {string[]}
+ */
+const INPUT_EXCEPTIONS_ACTUAL_ABOUT_BLANK_BASE = [ '#x' ];
+
+/**
+ * Given a URL, returns promise resolving to the downloaded URL contents parsed
+ * as JSON.
+ *
+ * @param {string} url URL to download.
+ *
+ * @return {Promise<*>} Promise resolving to result of parsed JSON.
+ */
+const fetchJSON = ( url ) =>
+	new Promise( ( resolve, reject ) => {
+		get( url, async ( response ) => {
+			if ( response.statusCode !== 200 ) {
+				return reject();
+			}
+
+			let string = '';
+
+			for await ( const chunk of response ) {
+				string += chunk.toString();
+			}
+
+			resolve( JSON.parse( string ) );
+		} );
+	} );
+
+/**
+ * Returns true if the given value is a test data item.
+ *
+ * @param {*} item Candidate to test.
+ *
+ * @return {boolean} Whether candidate is test data item.
+ */
+const isDataItem = ( item ) => item && item.input;
+
+/**
+ * Returns true if the given data item is expected to be used as the base
+ * parameter of a URL constructor.
+ *
+ * @param {Object} item Data item to test.
+ *
+ * @return {boolean} Whether data item has non-default base.
+ */
+const hasBase = ( item ) => item.base !== 'about:blank';
+
+/**
+ * Returns true if the given data item is included in the exception set.
+ *
+ * @param {Object} item Data item to test.
+ *
+ * @return {boolean} Whether data item is exception.
+ */
+const isException = ( item ) =>
+	INPUT_EXCEPTIONS_ACTUAL_ABOUT_BLANK_BASE.includes( item.input );
+
+/**
+ * Downloads data and writes output file.
+ *
+ * @param {string} [outFile] Optional output file.
+ */
+async function download( outFile = DEFAULT_OUT_FILE ) {
+	const data = await fetchJSON( DATA_URL );
+
+	const transformedData = data
+		.filter(
+			( item ) =>
+				isDataItem( item ) && ! hasBase( item ) && ! isException( item )
+		)
+		.map( ( item ) => ( {
+			input: item.input,
+			failure: item.failure,
+		} ) );
+
+	const file = fs.createWriteStream( outFile );
+	file.write( JSON.stringify( transformedData ) );
+	file.close();
+}
+
+module.exports = download;
+
+if ( ! module.parent ) {
+	try {
+		download();
+	} catch ( error ) {
+		process.stderr.write( error );
+		process.statusCode = 1;
+	}
+}

packages/url/src/is-url.native.js

@@ -1,16 +1,18 @@
-const URL_REGEXP = /^((([A-Za-z]{3,9}:(?:\/[\/]*)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+|(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)(?::\d{2,5})?((?:\/[\+~%\/\.\w\-_]*)?\??(?:[\-\+=&;%@\.\w_]*)#?(?:[\.\!\/\\\w]*))?)$/i;
 /**
- * Determines whether the given string looks like a URL.
- *
- * @param {string} url The string to scrutinise.
- *
- * @example
- * ```js
- * const isURL = isURL( 'https://wordpress.org' ); // true
- * ```
- *
- * @return {boolean} Whether or not it looks like a URL.
+ * External dependencies
+ */
+import { URL } from 'react-native-url-polyfill';
+
+/**
+ * @type {typeof import('./is-url').isURL}
  */
 export function isURL( url ) {
-	return URL_REGEXP.test( url );
+	// A URL can be considered value if the `URL` constructor is able to parse
+	// it. The constructor throws an error for an invalid URL.
+	try {
+		new URL( url );
+		return true;
+	} catch ( error ) {
+		return false;
+	}
 }

packages/url/src/test/fixtures/README.md

@@ -0,0 +1,25 @@
+# URL Fixtures
+
+The `@wordpress/url` module uses data from the [Web Platform Tests project](https://github.com/web-platform-tests/wpt) to verify expected behavior of its functionality as conforming to the [URL Living Standard](https://url.spec.whatwg.org/).
+
+This data is updated manually. To bring in the latest data, run the download script:
+
+```
+node packages/url/scripts/download-wpt-data.js
+```
+
+The Web Platform Tests URL data is made available under the 3-Clause BSD License:
+
+```
+# The 3-Clause BSD License
+
+Copyright 2019 web-platform-tests contributors
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+```

packages/url/src/test/fixtures/wpt-data.json

@@ -0,0 +1 @@
+[{"input":"https://test:@test"},{"input":"https://:@test"},{"input":"non-special://test:@test/x"},{"input":"non-special://:@test/x"},{"input":"lolscheme:x x#x x"},{"input":"file://example:1/","failure":true},{"input":"file://example:test/","failure":true},{"input":"file://example%/","failure":true},{"input":"file://[example]/","failure":true},{"input":"http://example.com/././foo"},{"input":"http://example.com/./.foo"},{"input":"http://example.com/foo/."},{"input":"http://example.com/foo/./"},{"input":"http://example.com/foo/bar/.."},{"input":"http://example.com/foo/bar/../"},{"input":"http://example.com/foo/..bar"},{"input":"http://example.com/foo/bar/../ton"},{"input":"http://example.com/foo/bar/../ton/../../a"},{"input":"http://example.com/foo/../../.."},{"input":"http://example.com/foo/../../../ton"},{"input":"http://example.com/foo/%2e"},{"input":"http://example.com/foo/%2e%2"},{"input":"http://example.com/foo/%2e./%2e%2e/.%2e/%2e.bar"},{"input":"http://example.com////../.."},{"input":"http://example.com/foo/bar//../.."},{"input":"http://example.com/foo/bar//.."},{"input":"http://example.com/foo"},{"input":"http://example.com/%20foo"},{"input":"http://example.com/foo%"},{"input":"http://example.com/foo%2"},{"input":"http://example.com/foo%2zbar"},{"input":"http://example.com/foo%2©zbar"},{"input":"http://example.com/foo%41%7a"},{"input":"http://example.com/foo\t‘%91"},{"input":"http://example.com/foo%00%51"},{"input":"http://example.com/(%28:%3A%29)"},{"input":"http://example.com/%3A%3a%3C%3c"},{"input":"http://example.com/foo\tbar"},{"input":"http://example.com\\\\foo\\\\bar"},{"input":"http://example.com/%7Ffp3%3Eju%3Dduvgw%3Dd"},{"input":"http://example.com/@asdf%40"},{"input":"http://example.com/你好你好"},{"input":"http://example.com/‥/foo"},{"input":"http://example.com//foo"},{"input":"http://example.com/‮/foo/‭/bar"},{"input":"http://www.google.com/foo?bar=baz#"},{"input":"http://www.google.com/foo?bar=baz# »"},{"input":"data:test# »"},{"input":"http://www.google.com"},{"input":"http://192.0x00A80001"},{"input":"http://www/foo%2Ehtml"},{"input":"http://www/foo/%2E/html"},{"input":"http://user:pass@/","failure":true},{"input":"http://%25DOMAIN:[email protected]/"},{"input":"http:\\\\www.google.com\\foo"},{"input":"http://foo:80/"},{"input":"http://foo:81/"},{"input":"httpa://foo:80/"},{"input":"http://foo:-80/","failure":true},{"input":"https://foo:443/"},{"input":"https://foo:80/"},{"input":"ftp://foo:21/"},{"input":"ftp://foo:80/"},{"input":"gopher://foo:70/"},{"input":"gopher://foo:443/"},{"input":"ws://foo:80/"},{"input":"ws://foo:81/"},{"input":"ws://foo:443/"},{"input":"ws://foo:815/"},{"input":"wss://foo:80/"},{"input":"wss://foo:81/"},{"input":"wss://foo:443/"},{"input":"wss://foo:815/"},{"input":"http:/example.com/"},{"input":"ftp:/example.com/"},{"input":"https:/example.com/"},{"input":"madeupscheme:/example.com/"},{"input":"file:/example.com/"},{"input":"ftps:/example.com/"},{"input":"gopher:/example.com/"},{"input":"ws:/example.com/"},{"input":"wss:/example.com/"},{"input":"data:/example.com/"},{"input":"javascript:/example.com/"},{"input":"mailto:/example.com/"},{"input":"http:example.com/"},{"input":"ftp:example.com/"},{"input":"https:example.com/"},{"input":"madeupscheme:example.com/"},{"input":"ftps:example.com/"},{"input":"gopher:example.com/"},{"input":"ws:example.com/"},{"input":"wss:example.com/"},{"input":"data:example.com/"},{"input":"javascript:example.com/"},{"input":"mailto:example.com/"},{"input":"http:@www.example.com"},{"input":"http:/@www.example.com"},{"input":"http://@www.example.com"},{"input":"http:a:[email protected]"},{"input":"http:/a:[email protected]"},{"input":"http://a:[email protected]"},{"input":"http://@pple.com"},{"input":"http::[email protected]"},{"input":"http:/:[email protected]"},{"input":"http://:[email protected]"},{"input":"http:/:@/www.example.com","failure":true},{"input":"http://user@/www.example.com","failure":true},{"input":"http:@/www.example.com","failure":true},{"input":"http:/@/www.example.com","failure":true},{"input":"http://@/www.example.com","failure":true},{"input":"https:@/www.example.com","failure":true},{"input":"http:a:b@/www.example.com","failure":true},{"input":"http:/a:b@/www.example.com","failure":true},{"input":"http://a:b@/www.example.com","failure":true},{"input":"http::@/www.example.com","failure":true},{"input":"http:a:@www.example.com"},{"input":"http:/a:@www.example.com"},{"input":"http://a:@www.example.com"},{"input":"http://[email protected]"},{"input":"http:@:www.example.com","failure":true},{"input":"http:/@:www.example.com","failure":true},{"input":"http://@:www.example.com","failure":true},{"input":"http://:@www.example.com"},{"input":"\u0000\u001b\u0004\u0012 http://example.com/\u001f \r "},{"input":"https://�","failure":true},{"input":"https://%EF%BF%BD","failure":true},{"input":"https://x/�?�#�"},{"input":"https://faß.ExAmPlE/"},{"input":"sc://faß.ExAmPlE/"},{"input":"https://x x:12","failure":true},{"input":"http://./"},{"input":"http://../"},{"input":"http://0..0x300/"},{"input":"http://[www.google.com]/","failure":true},{"input":"http://host/?'"},{"input":"notspecial://host/?'"},{"input":"about:/../"},{"input":"data:/../"},{"input":"javascript:/../"},{"input":"mailto:/../"},{"input":"sc://ñ.test/"},{"input":"sc://\u001f!\"$&'()*+,-.;<=>^_`{|}~/"},{"input":"sc://\u0000/","failure":true},{"input":"sc:// /","failure":true},{"input":"sc://%/"},{"input":"sc://@/","failure":true},{"input":"sc://te@s:t@/","failure":true},{"input":"sc://:/","failure":true},{"input":"sc://:12/","failure":true},{"input":"sc://[/","failure":true},{"input":"sc://\\/","failure":true},{"input":"sc://]/","failure":true},{"input":"sc:\\../"},{"input":"sc::[email protected]"},{"input":"wow:%NBD"},{"input":"wow:%1G"},{"input":"wow:￿"},{"input":"ftp://example.com%80/","failure":true},{"input":"ftp://example.com%A0/","failure":true},{"input":"https://example.com%80/","failure":true},{"input":"https://example.com%A0/","failure":true},{"input":"ftp://%e2%98%83"},{"input":"https://%e2%98%83"},{"input":"http://127.0.0.1:10100/relative_import.html"},{"input":"http://facebook.com/?foo=%7B%22abc%22"},{"input":"https://localhost:3000/[email protected]"},{"input":"h\tt\nt\rp://h\to\ns\rt:9\t0\n0\r0/p\ta\nt\rh?q\tu\ne\rry#f\tr\na\rg"},{"input":"http://foo.bar/baz?qux#foo\bbar"},{"input":"http://foo.bar/baz?qux#foo\"bar"},{"input":"http://foo.bar/baz?qux#foo<bar"},{"input":"http://foo.bar/baz?qux#foo>bar"},{"input":"http://foo.bar/baz?qux#foo`bar"},{"input":"https://0x.0x.0"},{"input":"https://0x100000000/test","failure":true},{"input":"https://256.0.0.1/test","failure":true},{"input":"file:///C%3A/"},{"input":"file:///C%7C/"},{"input":"file:\\\\//"},{"input":"file:\\\\\\\\"},{"input":"file:\\\\\\\\?fox"},{"input":"file:\\\\\\\\#guppy"},{"input":"file://spider///"},{"input":"file:\\\\localhost//"},{"input":"file:///localhost//cat"},{"input":"file://\\/localhost//cat"},{"input":"file://localhost//a//../..//"},{"input":"file://example.net/C:/"},{"input":"file://1.2.3.4/C:/"},{"input":"file://[1::8]/C:/"},{"input":"file:/C|/"},{"input":"file://C|/"},{"input":"file:"},{"input":"file:?q=v"},{"input":"file:#frag"},{"input":"https://[0::0::0]","failure":true},{"input":"https://[0:.0]","failure":true},{"input":"https://[0:0:]","failure":true},{"input":"https://[0:1:2:3:4:5:6:7.0.0.0.1]","failure":true},{"input":"https://[0:1.00.0.0.0]","failure":true},{"input":"https://[0:1.290.0.0.0]","failure":true},{"input":"https://[0:1.23.23]","failure":true},{"input":"http://?","failure":true},{"input":"http://#","failure":true},{"input":"sc://ñ"},{"input":"sc://ñ?x"},{"input":"sc://ñ#x"},{"input":"sc://?"},{"input":"sc://#"},{"input":"tftp://foobar.com/someconfig;mode=netascii"},{"input":"telnet://user:[email protected]:23/"},{"input":"ut2004://10.10.10.10:7777/Index.ut2"},{"input":"redis://foo:bar@somehost:6379/0?baz=bam&qux=baz"},{"input":"rsync://foo@host:911/sup"},{"input":"git://github.com/foo/bar.git"},{"input":"irc://myserver.com:6999/channel?passwd"},{"input":"dns://fw.example.org:9999/foo.bar.org?type=TXT"},{"input":"ldap://localhost:389/ou=People,o=JNDITutorial"},{"input":"git+https://github.com/foo/bar"},{"input":"urn:ietf:rfc:2648"},{"input":"tag:[email protected],2001:foo/bar"},{"input":"non-special://%E2%80%A0/"},{"input":"non-special://H%4fSt/path"},{"input":"non-special://[1:2:0:0:5:0:0:0]/"},{"input":"non-special://[1:2:0:0:0:0:0:3]/"},{"input":"non-special://[1:2::3]:80/"},{"input":"non-special://[:80/","failure":true},{"input":"blob:https://example.com:443/"},{"input":"blob:d3958f5c-0777-0845-9dcf-2cb28783acaf"},{"input":"http://0177.0.0.0189"},{"input":"http://0x7f.0.0.0x7g"},{"input":"http://0X7F.0.0.0X7G"},{"input":"http://[::127.0.0.0.1]","failure":true},{"input":"http://[0:1:0:1:0:1:0:1]"},{"input":"http://[1:0:1:0:1:0:1:0]"},{"input":"http://example.org/test?\""},{"input":"http://example.org/test?#"},{"input":"http://example.org/test?<"},{"input":"http://example.org/test?>"},{"input":"http://example.org/test?⌣"},{"input":"http://example.org/test?%23%23"},{"input":"http://example.org/test?%GH"},{"input":"http://example.org/test?a#%EF"},{"input":"http://example.org/test?a#%GH"},{"input":"a","failure":true},{"input":"a/","failure":true},{"input":"a//","failure":true},{"input":"http://example.org/test?a#b\u0000c"}]

packages/url/src/test/index.native.js

@@ -2,3 +2,26 @@
  * Internal dependencies
  */
 import './index.test';
+
+jest.mock( './fixtures/wpt-data.json', () => {
+	const data = require.requireActual( './fixtures/wpt-data.json' );
+
+	/**
+	 * Test items to exclude by input. Ideally this should be empty, but are
+	 * necessary by non-spec-conformance of the Native implementations.
+	 * Specifically, the React Native implementation uses an implementation of
+	 * WHATWG URL without full Unicode support.
+	 *
+	 * @type {string[]}
+	 */
+	const URL_EXCEPTIONS = [
+		'https://�',
+		'https://%EF%BF%BD',
+		'ftp://example.com%80/',
+		'ftp://example.com%A0/',
+		'https://example.com%80/',
+		'https://example.com%A0/',
+	];
+
+	return data.filter( ( { input } ) => ! URL_EXCEPTIONS.includes( input ) );
+} );

packages/url/src/test/index.test.js

@@ -28,36 +28,15 @@ import {
 	filterURLForDisplay,
 	cleanForSlug,
 } from '../';
+import wptData from './fixtures/wpt-data';
 
 describe( 'isURL', () => {
-	it.each( [
-		[ 'http://wordpress.org' ],
-		[ 'https://wordpress.org' ],
-		[ 'HTTPS://WORDPRESS.ORG' ],
-		[ 'https://wordpress.org/./foo' ],
-		[ 'https://wordpress.org/path?query#fragment' ],
-		[ 'https://localhost/foo#bar' ],
-		[ 'https:///localhost/foo#bar' ],
-		[ 'mailto:[email protected]' ],
-		[ 'ssh://user:[email protected]:8080' ],
-		[ 'file:///localfolder/file.mov' ],
-		[ 'file:/localfolder/file.mov' ],
-	] )( 'valid (true): %s', ( url ) => {
-		expect( isURL( url ) ).toBe( true );
-	} );
-
-	it.each( [
-		[ 'http://word press.org' ],
-		[ 'http://wordpress.org:port' ],
-		[ 'http://[wordpress.org]/' ],
-		[ 'HTTP: HyperText Transfer Protocol' ],
-		[ 'URLs begin with a http:// prefix' ],
-		[ 'Go here: http://wordpress.org' ],
-		[ 'http://' ],
-		[ '' ],
-	] )( 'invalid (false): %s', ( url ) => {
-		expect( isURL( url ) ).toBe( false );
-	} );
+	it.each( wptData.map( ( { input, failure } ) => [ input, !! failure ] ) )(
+		'%s',
+		( input, isFailure ) => {
+			expect( isURL( input ) ).toBe( ! isFailure );
+		}
+	);
 } );
 
 describe( 'isEmail', () => {