Stschr/lang matcher dash parser (#4036)

* Implemented a 'matcher' to sanitize all @lang attributes parsed from manifests into a bcp-47 complinat format.
Added unit tests to check correct operation of 3-letter code to 2-letter conversion for both AdaptationSet@lang and Label@lang.

package.json

@@ -57,6 +57,8 @@
         "yargs": "16.0.3"
     },
     "dependencies": {
+        "bcp-47-match": "^1.0.3",
+        "bcp-47-normalize": "^1.1.1",
         "codem-isoboxer": "0.3.6",
         "es6-promise": "^4.2.8",
         "fast-deep-equal": "2.0.1",

src/dash/constants/DashConstants.js

@@ -142,6 +142,8 @@ class DashConstants {
         this.ORIGINAL_MPD_ID = 'mpdId';
         this.WALL_CLOCK_TIME = 'wallClockTime';
         this.PRESENTATION_TIME = 'presentationTime';
+        this.LABEL = 'Label';
+        this.GROUP_LABEL = 'GroupLabel';
         this.CONTENT_STEERING = 'ContentSteering';
         this.CONTENT_STEERING_AS_ARRAY = 'ContentSteering_asArray';
         this.DEFAULT_SERVICE_LOCATION = 'defaultServiceLocation';

src/dash/models/DashManifestModel.js

@@ -215,8 +215,7 @@ function DashManifestModel() {
         let lang = '';
 
         if (adaptation && adaptation.hasOwnProperty(DashConstants.LANG)) {
-            //Filter out any other characters not allowed according to RFC5646
-            lang = adaptation.lang.replace(/[^A-Za-z0-9-]/g, '');
+            lang = adaptation.lang;
         }
 
         return lang;

src/dash/parser/DashParser.js

@@ -35,6 +35,7 @@ import StringMatcher from './matchers/StringMatcher';
 import DurationMatcher from './matchers/DurationMatcher';
 import DateTimeMatcher from './matchers/DateTimeMatcher';
 import NumericMatcher from './matchers/NumericMatcher';
+import LangMatcher from './matchers/LangMatcher';
 import RepresentationBaseValuesMap from './maps/RepresentationBaseValuesMap';
 import SegmentValuesMap from './maps/SegmentValuesMap';
 
@@ -56,6 +57,7 @@ function DashParser(config) {
             new DurationMatcher(),
             new DateTimeMatcher(),
             new NumericMatcher(),
+            new LangMatcher(),
             new StringMatcher()   // last in list to take precedence over NumericMatcher
         ];
 

src/dash/parser/matchers/LangMatcher.js

@@ -0,0 +1,71 @@
+/**
+ * The copyright in this software is being made available under the BSD License,
+ * included below. This software may be subject to other third party and contributor
+ * rights, including patent rights, and no such rights are granted under this license.
+ *
+ * Copyright (c) 2013, Dash Industry Forum.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *  * Redistributions of source code must retain the above copyright notice, this
+ *  list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *  this list of conditions and the following disclaimer in the documentation and/or
+ *  other materials provided with the distribution.
+ *  * Neither the name of Dash Industry Forum nor the names of its
+ *  contributors may be used to endorse or promote products derived from this software
+ *  without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS AND ANY
+ *  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ *  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ *  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ *  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ *  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+/**
+ * @classdesc Matches and converts any ISO 639 language tag to BCP-47 language tags
+ */
+import BaseMatcher from './BaseMatcher';
+import DashConstants from '../../constants/DashConstants';
+var bcp47Normalize = require('bcp-47-normalize')
+
+class LangMatcher extends BaseMatcher {
+    constructor() {
+        super(
+            (attr, nodeName) => {
+                const stringAttrsInElements = {
+                    [DashConstants.ADAPTATION_SET]:                 [ DashConstants.LANG ],
+                    [DashConstants.REPRESENTATION]:                 [ DashConstants.LANG ],
+                    [DashConstants.CONTENT_COMPONENT]:              [ DashConstants.LANG ],
+                    [DashConstants.LABEL]:                          [ DashConstants.LANG ],
+                    [DashConstants.GROUP_LABEL]:                    [ DashConstants.LANG ]
+                    // still missing from 23009-1: Preselection@lang, ProgramInformation@lang
+                };
+                if (stringAttrsInElements.hasOwnProperty(nodeName)) {
+                    let attrNames = stringAttrsInElements[nodeName];
+                    if (attrNames !== undefined) {
+                        return attrNames.indexOf(attr.name) >= 0;
+                    } else {
+                        return false;
+                    }
+                }
+                return false;
+            },
+            str => {
+                let lang = bcp47Normalize(str);
+                if (lang !== undefined) {
+                    return lang;
+                }
+                return String(str);
+            }
+        );
+    }
+}
+
+export default LangMatcher;

test/unit/dash.DashParser.js

@@ -1,5 +1,8 @@
 import DashParser from '../../src/dash/parser/DashParser';
 import DebugMock from './mocks/DebugMock';
+import DashManifestModel from '../../src/dash/models/DashManifestModel';
+
+import ErrorHandlerMock from './mocks/ErrorHandlerMock';
 
 const expect = require('chai').expect;
 const fs = require('fs');
@@ -8,6 +11,8 @@ const jsdom = require('jsdom').JSDOM;
 const context = {};
 
 let dashParser = DashParser(context).create({debug: new DebugMock()});
+const errorHandlerMock = new ErrorHandlerMock();
+const dashManifestModel = DashManifestModel(context).getInstance();
 
 describe('DashParser', function () {
 
@@ -36,4 +41,38 @@ describe('DashParser', function () {
         let manifest = fs.readFileSync(__dirname + '/data/dash/manifest_error.xml', 'utf8');
         expect(dashParser.parse.bind(manifest)).to.be.throw('parsing the manifest failed');
     });
+
+    it('should return an Object when parse is called with correct data', function () {
+        let manifest = fs.readFileSync(__dirname + '/data/dash/manifest.xml', 'utf8');
+        expect(dashParser.parse.bind(manifest)).to.be.instanceOf(Object);   // jshint ignore:line
+    });
+
+    describe('DashParser matchers', function () {
+        beforeEach(function () {
+            dashManifestModel.setConfig({
+                errHandler: errorHandlerMock
+            });
+        });
+
+        let manifest = fs.readFileSync(__dirname + '/data/dash/manifest.xml', 'utf8');
+
+        it('should return normalized language tag', function () {
+            let parsedMpd = dashParser.parse(manifest);
+            let audioAdaptationsArray = dashManifestModel.getAdaptationsForType(parsedMpd, 0, 'audio');
+
+            expect(audioAdaptationsArray).to.be.instanceOf(Array);      // jshint ignore:line
+            expect(audioAdaptationsArray.length).to.equal(1);           // jshint ignore:line
+            expect(dashManifestModel.getLanguageForAdaptation(audioAdaptationsArray[0])).to.equal('es');  // jshint ignore:line
+        });
+
+        it('should return normalized language tages for labels on AdaptationSets', function () {
+            let parsedMpd = dashParser.parse(manifest);
+            let audioAdaptation = dashManifestModel.getAdaptationsForType(parsedMpd, 0, 'audio')[0];
+            let labelArray = dashManifestModel.getLabelsForAdaptation(audioAdaptation);
+
+            expect(labelArray).to.be.instanceOf(Array);      // jshint ignore:line
+            expect(labelArray.length).to.equal(2);           // jshint ignore:line
+            expect(labelArray[1].lang).to.equal('fr');       // jshint ignore:line
+        });
+    });
 });

test/unit/data/dash/manifest.xml

@@ -14,7 +14,9 @@
             <Representation id="bbb_30fps_768x432_1500k" codecs="avc1.64001e" bandwidth="1883700" width="768" height="432" frameRate="30" sar="1:1" scanType="progressive"/>
             <Representation id="bbb_30fps_3840x2160_12000k" codecs="avc1.640033" bandwidth="14931538" width="3840" height="2160" frameRate="30" sar="1:1" scanType="progressive"/>
         </AdaptationSet>
-        <AdaptationSet mimeType="audio/mp4" contentType="audio" subsegmentAlignment="true" subsegmentStartsWithSAP="1">
+        <AdaptationSet mimeType="audio/mp4" contentType="audio" subsegmentAlignment="true" subsegmentStartsWithSAP="1" lang="spa">
+            <Label lang="en">English Label</Label>
+            <Label lang="fre">French Label</Label>
             <Accessibility schemeIdUri="urn:tva:metadata:cs:AudioPurposeCS:2007" value="6"/>
             <Role schemeIdUri="urn:mpeg:dash:role:2011" value="main"/>
             <SegmentTemplate duration="192512" timescale="48000" media="$RepresentationID$/$RepresentationID$_$Number$.m4a" startNumber="1" initialization="$RepresentationID$/$RepresentationID$_0.m4a"/>