Add AutoExtensionImporter

packages/optimizer/lib/RuntimeHostHelper.js

@@ -0,0 +1,44 @@
+/**
+ * Copyright 2020 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+const {
+  AMP_CACHE_HOST,
+  appendRuntimeVersion,
+} = require('./AmpConstants.js');
+
+function calculateHost(params) {
+  let ampUrlPrefix = params.ampUrlPrefix || AMP_CACHE_HOST;
+  if (params.ampRuntimeVersion && !params.ampUrlPrefix) {
+    ampUrlPrefix = appendRuntimeVersion(ampUrlPrefix, params.ampRuntimeVersion);
+  }
+  let dynamicAmpUrlPrefix = ampUrlPrefix;
+  if (params.rewriteDynamicComponents === false) {
+    dynamicAmpUrlPrefix = AMP_CACHE_HOST;
+    if (params.ampRuntimeVersion) {
+      dynamicAmpUrlPrefix = appendRuntimeVersion(dynamicAmpUrlPrefix, params.ampRuntimeVersion);
+    }
+  }
+  return {
+    dynamicAmpUrlPrefix,
+    ampUrlPrefix,
+  };
+}
+
+module.exports = {
+  calculateHost,
+};
+

packages/optimizer/lib/transformers/AutoExtensionImporter.js

@@ -0,0 +1,269 @@
+/**
+ * Copyright 2020 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+const {findMetaViewport} = require('../HtmlDomHelper');
+const {calculateHost} = require('../RuntimeHostHelper');
+const validatorRules = require('@ampproject/toolbox-validator-rules');
+
+const BIND_SHORT_FORM_PREFIX = 'bind';
+
+// Some AMP component don't bring their own tag, but enable new attributes on other
+// elements. Most are included in the AMP validation rules, but some are not. These
+// need to be defined manually here.
+const manualAttributeToExtensionMapping = new Map([
+  ['mask', 'amp-inputmask'],
+  ['lightbox', 'amp-lightbox-gallery'],
+]);
+
+/**
+ * Extension Auto Importer - this transformer auto imports all missing AMP extensions.
+ *
+ * The importer analyzes the HTML source code and identifies missing AMP extension imports
+ * using multiple strategies:
+ *
+ * - use validation rules to map used AMP tags to required AMP extensions.
+ * - use validation rules to map used AMP attributes to required AMP extensions.
+ * - manually specifiy attribute to extension mappings if this information is not available in the
+ *   validation rules.
+ * - mnullay implement AMP extension detection for a few corner cases.
+ *
+ * This importer also enables a shortcode `bindtext` instead of `data-amp-bind-text` for specifying
+ * AMP bindings when the square bracket notation (`[text]`) is not available. To avoid accidently
+ * rewriting non-AMP attributes, the transformer uses the AMP validation rules to only rename bindable
+ * attributes as specified in the validation rules.
+ */
+class AutoExtensionImporter {
+  constructor(config) {
+    this.enabled = config.autoExtensionImport !== 'false' && config.autoExtensionImport !== false;
+    this.log_ = config.log.tag('AutoExtensionImporter');
+
+    // We use the validation rules to infer extension imports. The rules are downloaded once and for
+    // efficency, we initially extract all needed rules
+    this.extensionSpec = validatorRules.fetch().then((rules) => {
+      // Map extension names to more info
+      const extensionsMap = new Map(rules.extensions.map((ext) => [ext.name, {
+        name: ext.name,
+        type: ext.extensionType === 'CUSTOM_TEMPLATE' ? 'custom-template' : 'custom-element',
+        version: ext.version.filter((v) => v !== 'latest'),
+      }]));
+      // Maps tags (e.g. amp-state) to their extension (e.g. amp-bind)
+      const tagToExtensionsMapping = new Map();
+      // Maps tags to their extension specific allowed attributes
+      // (e.g. amp-img => amp-fx => amp-fx-collection)
+      const tagToAttributeMapping = new Map();
+      // Maps tags to their bindable attributes (e.g. div => text)
+      const tagToBindAttributeMapping = new Map();
+
+      // Iterate over all available tags
+      for (const tag of rules.tags) {
+        const tagName = tag.tagName.toLowerCase();
+
+        // Map amp tags to their required extension(s)
+        if (tagName.startsWith('amp-')) {
+          tagToExtensionsMapping.set(
+              tagName,
+              tag.requiresExtension,
+          );
+        }
+        // Hack: fix missing attribute dependencies (e.g. amp-img => lightbox => amp-lightbox-gallery)
+        tag.attrs.filter((a) => manualAttributeToExtensionMapping.has(a.name))
+            .forEach((a) => {
+              a.requiresExtension = [manualAttributeToExtensionMapping.get(a.name)];
+            });
+
+        // Map attributes to tags and extensions (e.g. amp-img => amp-fx => amp-fx-collection)
+        tag.attrs.filter((a) => a.requiresExtension && a.requiresExtension.length > 0)
+            .forEach((a) => {
+              let attributeMapping = tagToAttributeMapping.get(tagName);
+              if (!attributeMapping) {
+                attributeMapping = [];
+                tagToAttributeMapping.set(tagName, attributeMapping);
+              }
+              attributeMapping.push(a);
+            });
+
+        // Map tags to bindable attributes which are named link `[text]`
+        const bindableAttributes = tag.attrs.filter((a) => a.name.startsWith('['))
+            .map((a) => a.name.substring(1, a.name.length - 1));
+        tagToBindAttributeMapping.set(tagName, new Set(bindableAttributes));
+      }
+      return {
+        extensionsMap,
+        tagToExtensionsMapping,
+        tagToAttributeMapping,
+        tagToBindAttributeMapping,
+      };
+    });
+  }
+
+  async transform(tree, params) {
+    if (!this.enabled) {
+      return;
+    }
+    const html = tree.root.firstChildByTag('html');
+    const head = html.firstChildByTag('head');
+    if (!head) return;
+
+    // Extensions which need to be imported
+    const extensionsToImport = new Set();
+    // Keep track of existing extensions imports to avoid duplicates
+    const existingImports = new Set();
+
+    // Some AMP components need to be detected in the head (e.g. amp-access)
+    await this.findExtensionsToImportInHead_(head, extensionsToImport, existingImports);
+
+    // Most AMP components can be detected in the body
+    const body = html.firstChildByTag('body');
+    await this.findExtensionsToImportInBody_(body, extensionsToImport);
+
+    if (extensionsToImport.length === 0) {
+      // Nothing to do
+      return;
+    }
+
+    // We use this for adding new import elements to the header
+    const referenceNode = findMetaViewport(head);
+
+    // Support custom runtime URLs
+    const host = calculateHost(params);
+    for (const extensionName of extensionsToImport) {
+      if (!existingImports.has(extensionName)) {
+        const extension = (await this.extensionSpec).extensionsMap.get(extensionName.trim());
+        this.log_.debug('auto importing', extensionName);
+        // Use the latest version by default
+        const version = extension.version[extension.version.length - 1];
+        const extensionImportAttribs = {
+          'async': '',
+          'src': `${host.ampUrlPrefix}/v0/${extensionName}-${version}.js`,
+        };
+        extensionImportAttribs[extension.type] = extensionName;
+        const extensionImport = tree.createElement('script', extensionImportAttribs);
+        head.insertAfter(extensionImport, referenceNode);
+      }
+    }
+  }
+
+  findExtensionsToImportInHead_(head, extensionsToImport, existingImports) {
+    let node = head.firstChild;
+    while (node) {
+      // Detect any existing extension imports
+      const customElement = this.getCustomElement_(node);
+      if (customElement) {
+        existingImports.add(customElement);
+      }
+      // Explicitly detect amp-access via the script tag in the header
+      if (node.tagName === 'script' && node.attribs['id'] === 'amp-access') {
+        extensionsToImport.add('amp-access');
+      }
+      node = node.nextSibling;
+    }
+  }
+
+  async findExtensionsToImportInBody_(body, extensionsToImport) {
+    let node = body;
+    const promises = [];
+    while (node !== null) {
+      if (node.tagName) {
+        promises.push(this.addRequiredExtensionByTag_(extensionsToImport, node));
+        promises.push(this.addRequiredExtensionByAttributes_(extensionsToImport, node));
+      }
+      node = node.nextNode();
+    }
+    return Promise.all(promises);
+  }
+
+  async addRequiredExtensionByTag_(allRequiredExtensions, node) {
+    const extensionSpec = (await this.extensionSpec);
+    // Check for required extensions by tag name
+    const requiredExtensions = extensionSpec.tagToExtensionsMapping.get(node.tagName);
+    if (requiredExtensions) {
+      requiredExtensions.forEach((ext) => allRequiredExtensions.add(ext));
+    }
+    // Add custom templates (e.g. amp-mustache)
+    if (node.tagName === 'template' && node.attribs.type) {
+      allRequiredExtensions.add(node.attribs.type);
+    }
+  }
+
+  async addRequiredExtensionByAttributes_(allRequiredExtensions, node) {
+    if (!node.tagName || !node.attribs) {
+      return;
+    }
+    // Look for element attributes indicating AMP components (e.g. amp-fx)
+    const extensionSpec = await this.extensionSpec;
+    const tagToAttributeMapping = extensionSpec.tagToAttributeMapping;
+    const attributesForTag = tagToAttributeMapping.get(node.tagName);
+    if (attributesForTag) {
+      attributesForTag.forEach((attribute) => {
+        if (node.attribs[attribute.name] !== undefined) {
+          attribute.requiresExtension.forEach((ext) => {
+            allRequiredExtensions.add(ext);
+          });
+        }
+      });
+    }
+    // Check for amp-bind attribute bindings
+    const tagToBindAttributeMapping = extensionSpec.tagToBindAttributeMapping;
+    const attributeNames = Object.keys(node.attribs);
+    if (attributeNames.some((a) => a.startsWith('[') || a.startsWith('data-amp-bind'))) {
+      allRequiredExtensions.add('amp-bind');
+    }
+    // Rewrite short-form `bindtext` to `data-amp-bind-text`
+    // to avoid false-positives we check for each tag only the
+    // supported bindable attributes (e.g. for a div only bindtext, but not bindvalue).
+    const ampBindAttrs = tagToBindAttributeMapping.get(node.tagName);
+    // true if we need to import amp-bind
+    let usesAmpBind = false;
+    for (const attributeName of attributeNames) {
+      if (!attributeName.startsWith(BIND_SHORT_FORM_PREFIX)) {
+        continue;
+      }
+      const attributeNameWithoutBindPrefix =
+        attributeName.substring(BIND_SHORT_FORM_PREFIX.length);
+
+      // Rename attribute from bindx to data-amp-bind-x
+      if (ampBindAttrs.has(attributeNameWithoutBindPrefix)) {
+        const newAttributeName = `data-amp-bind-${attributeNameWithoutBindPrefix}`;
+        node.attribs[newAttributeName] = node.attribs[attributeName];
+        delete node.attribs[attributeName];
+        usesAmpBind = true;
+      }
+    }
+    if (usesAmpBind) {
+      allRequiredExtensions.add('amp-bind');
+    }
+  }
+
+  getCustomElement_(scriptNode) {
+    if (scriptNode.tagName !== 'script') {
+      return '';
+    }
+    let customElement = scriptNode.attribs['custom-element'] ||
+      scriptNode.attribs['custom-template'] ||
+      '';
+    if (!customElement) {
+      return '';
+    }
+    customElement = customElement.toLowerCase();
+    if (!customElement.startsWith('amp-')) {
+      return '';
+    }
+    return customElement;
+  }
+}
+
+module.exports = AutoExtensionImporter;

packages/optimizer/lib/transformers/RewriteAmpUrls.js

@@ -18,9 +18,9 @@
 const {
   AMP_CACHE_HOST,
   AMP_DYNAMIC_COMPONENTS,
-  appendRuntimeVersion,
 } = require('../AmpConstants.js');
 const {findMetaViewport} = require('../HtmlDomHelper');
+const {calculateHost} = require('../RuntimeHostHelper');
 
 /**
  * RewriteAmpUrls - rewrites AMP runtime URLs.
@@ -57,17 +57,7 @@ class RewriteAmpUrls {
     const head = html.firstChildByTag('head');
     if (!head) return;
 
-    let ampUrlPrefix = params.ampUrlPrefix || AMP_CACHE_HOST;
-    if (params.ampRuntimeVersion && !params.ampUrlPrefix) {
-      ampUrlPrefix = appendRuntimeVersion(ampUrlPrefix, params.ampRuntimeVersion);
-    }
-    let dynamicAmpUrlPrefix = ampUrlPrefix;
-    if (params.rewriteDynamicComponents === false) {
-      dynamicAmpUrlPrefix = AMP_CACHE_HOST;
-      if (params.ampRuntimeVersion) {
-        dynamicAmpUrlPrefix = appendRuntimeVersion(dynamicAmpUrlPrefix, params.ampRuntimeVersion);
-      }
-    }
+    const host = calculateHost(params);
 
     let node = head.firstChild;
     let referenceNode = findMetaViewport(head);
@@ -76,12 +66,12 @@ class RewriteAmpUrls {
       if (node.tagName === 'script' && this._usesAmpCacheUrl(node.attribs.src)) {
         const isDynamicComponent = this._isDynamicComponent(node);
         node.attribs.src = this._replaceUrl(node.attribs.src,
-          isDynamicComponent ? dynamicAmpUrlPrefix : ampUrlPrefix);
+          isDynamicComponent ? host.dynamicAmpUrlPrefix : host.ampUrlPrefix);
         referenceNode = this._addPreload(tree, head, referenceNode, node.attribs.src, 'script');
       } else if (node.tagName === 'link' &&
                   node.attribs.rel === 'stylesheet' &&
                   this._usesAmpCacheUrl(node.attribs.href)) {
-        node.attribs.href = this._replaceUrl(node.attribs.href, ampUrlPrefix);
+        node.attribs.href = this._replaceUrl(node.attribs.href, host.ampUrlPrefix);
         referenceNode = this._addPreload(tree, head, referenceNode, node.attribs.href, 'style');
       }
       node = node.nextSibling;

packages/optimizer/spec/transformers/valid/AutoExtensionImporter/amp-bind-bindX/expected_output.html

@@ -0,0 +1 @@
+<!DOCTYPE html><html ⚡=""><head><meta charset="utf-8"><title>My AMP Page</title><link rel="canonical" href="self.html"><meta name="viewport" content="width=device-width,minimum-scale=1"><script async="" src="https://cdn.ampproject.org/v0/amp-bind-0.1.js" custom-element="amp-bind"></script><style amp-boilerplate="">body{-webkit-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-moz-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-ms-animation:-amp-start 8s steps(1,end) 0s 1 normal both;animation:-amp-start 8s steps(1,end) 0s 1 normal both}@-webkit-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-moz-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-ms-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-o-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}</style><noscript><style amp-boilerplate>body{-webkit-animation:none;-moz-animation:none;-ms-animation:none;animation:none}</style></noscript><script async="" src="https://cdn.ampproject.org/v0.js"></script></head><body><div data-amp-bind-text="greeting"></div><button on="tap:AMP.setState({ greeting: 'hello world' })">Say hello</button></body></html>

packages/optimizer/spec/transformers/valid/AutoExtensionImporter/amp-bind-bindX/input.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html ⚡>
+<head>
+  <meta charset="utf-8">
+  <title>My AMP Page</title>
+  <link rel="canonical" href="self.html" />
+  <meta name="viewport" content="width=device-width,minimum-scale=1">
+  <style amp-boilerplate>body{-webkit-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-moz-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-ms-animation:-amp-start 8s steps(1,end) 0s 1 normal both;animation:-amp-start 8s steps(1,end) 0s 1 normal both}@-webkit-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-moz-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-ms-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-o-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}</style><noscript><style amp-boilerplate>body{-webkit-animation:none;-moz-animation:none;-ms-animation:none;animation:none}</style></noscript>
+  <script async src="https://cdn.ampproject.org/v0.js"></script>
+</head>
+<body>
+  <div bindtext="greeting"></div>
+  <button on="tap:AMP.setState({ greeting: 'hello world' })">Say hello</button>
+</body>
+</html>

packages/optimizer/spec/transformers/valid/AutoExtensionImporter/amp-bind-data-amp-bind/expected_output.html

@@ -0,0 +1 @@
+<!DOCTYPE html><html ⚡=""><head><meta charset="utf-8"><title>My AMP Page</title><link rel="canonical" href="self.html"><meta name="viewport" content="width=device-width,minimum-scale=1"><script async="" src="https://cdn.ampproject.org/v0/amp-bind-0.1.js" custom-element="amp-bind"></script><style amp-boilerplate="">body{-webkit-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-moz-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-ms-animation:-amp-start 8s steps(1,end) 0s 1 normal both;animation:-amp-start 8s steps(1,end) 0s 1 normal both}@-webkit-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-moz-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-ms-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-o-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}</style><noscript><style amp-boilerplate>body{-webkit-animation:none;-moz-animation:none;-ms-animation:none;animation:none}</style></noscript><script async="" src="https://cdn.ampproject.org/v0.js"></script></head><body><div data-amp-bind-text="greeting"></div><button on="tap:AMP.setState({ greeting: 'hello world' })">Say hello</button></body></html>

packages/optimizer/spec/transformers/valid/AutoExtensionImporter/amp-bind-data-amp-bind/input.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html ⚡>
+<head>
+  <meta charset="utf-8">
+  <title>My AMP Page</title>
+  <link rel="canonical" href="self.html" />
+  <meta name="viewport" content="width=device-width,minimum-scale=1">
+  <style amp-boilerplate>body{-webkit-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-moz-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-ms-animation:-amp-start 8s steps(1,end) 0s 1 normal both;animation:-amp-start 8s steps(1,end) 0s 1 normal both}@-webkit-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-moz-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-ms-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-o-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}</style><noscript><style amp-boilerplate>body{-webkit-animation:none;-moz-animation:none;-ms-animation:none;animation:none}</style></noscript>
+  <script async src="https://cdn.ampproject.org/v0.js"></script>
+</head>
+<body>
+  <div data-amp-bind-text="greeting"></div>
+  <button on="tap:AMP.setState({ greeting: 'hello world' })">Say hello</button>
+</body>
+</html>