- PLAT-5939 unexpectedly large pts to dts delay (#233)

* - PLAT-5939. added diagnostic messages from converter - setenv.sh debug/release build fix - json serialization fix when there are no keyFrameDurations present (e.g audio only) * - 1.0-PLAT-5939 built node addon on centos * - PLAT-5939 replace '~ ' with user home dir at the end of config file merge * - typo * - 1.0-PLAT-5939 * PLAT-5939 built centos addon files
kaltura · Aug 10, 2016 · 144ad10 · 144ad10
1 parent bb0a89c
commit 144ad10
Show file tree

Hide file tree

Showing 14 changed files with 105 additions and 26 deletions.
diff --git a/bin/darwin/FormatConverter.node b/bin/darwin/FormatConverter.node
diff --git a/bin/darwin/FormatConverter.node.debug b/bin/darwin/FormatConverter.node.debug
diff --git a/bin/linux/FormatConverter.node b/bin/linux/FormatConverter.node
diff --git a/bin/linux/FormatConverter.node.debug b/bin/linux/FormatConverter.node.debug
diff --git a/common/Configuration.js b/common/Configuration.js
@@ -13,7 +13,6 @@ module.exports = (function(){
 
     var configTemplateContent = fs.readFileSync(path.join(__dirname, './config/config.json.template'), 'utf8');
     var updatedConfigContent = configTemplateContent.replace('@HOSTNAME@', machineName);
-    updatedConfigContent= updatedConfigContent.replace(/~/g,hostname.homedir());
 
     var configObj = JSON.parse(updatedConfigContent);
 
@@ -43,7 +42,7 @@ module.exports = (function(){
         }
     }
 
-    fs.writeFileSync(path.join(__dirname, './config/config.json'), JSON.stringify(configObj, null, 2));
+    fs.writeFileSync(path.join(__dirname, './config/config.json'), JSON.stringify(configObj, null, 2).replace(/~/g,hostname.homedir()));
 
     var nconf = require('nconf');
 

diff --git a/lib/MP4WriteStream.js b/lib/MP4WriteStream.js
@@ -12,6 +12,7 @@ var config = require('./../common/Configuration');
 var qio = require('q-io/fs');
 var persistenceFormat = require('./../common/PersistenceFormat');
 var loggerModule = require('../common/logger');
+var _ = require('underscore');
 
 function getArchAndPlatform()
 {
@@ -146,6 +147,11 @@ var onFinish = function() {
         that.savedChunk.end();
     }
 
+    if(!that.chunks.length){
+        onError.call(that,'empty ts chunk!');
+        return;
+    }
+
     that.logger.trace("onFinish. Create converter ");
     var converter = new nativeTS2MP4.TS2MP4Convertor();
 

diff --git a/lib/entry/FlavorDownloader.js b/lib/entry/FlavorDownloader.js
@@ -305,17 +305,30 @@ var FlavorDownloader = (function() {
                 var pGlob = Q.denodeify(glob);
                 return pGlob(path.join(that.destFolderPath, '*.mp4'))
                     .then(function(downloadedChunks) {
-                        _.each(downloadedChunks, function(c) {
+                        var checkCrashedFiles = [];
+                        _.each(downloadedChunks, function (c) {
                             var fileName = path.basename(c);
-                            if(that.playlistGenerator.checkFileExists(fileName)) {
-                                that.logger.trace("Adding %s to downloadedChunks",fileName);
-                                var tsFileName = persistenceFormat.getTSChunknameFromMP4FileName(fileName);
+                            var tsFileName = persistenceFormat.getTSChunknameFromMP4FileName(fileName);
+                            if (that.playlistGenerator.checkFileExists(fileName)) {
+                                that.logger.trace("Adding %s to downloadedChunks", fileName);
                                 that.downloadedChunks[tsFileName] = true;
                                 that.downloadedChunks[fileName] = tsFileName;
+                            } else {
+                                // test against 0 length file - guard against endless looping caused
+                                // by crashing while attempting to convert chunk
+                                checkCrashedFiles.push(qfs.stat(c).then(function (stats) {
+                                    if (!stats.size) {
+                                        that.logger.warn("Zero size file! Adding %s to chunksToDownload", fileName);
+                                        that.chunksToDownload[tsFileName] = true;
+                                    }
+                                }));
                             }
                         });
+                        if (checkCrashedFiles.length) {
+                            return Q.allSettled(checkCrashedFiles);
+                        }
                     });
-            })
+                })
             .then(function() {
                 return downloadIndefinitely.call(that, onIterationEndCallback);
             })

diff --git a/lib/playlistGenerator/MixFilterClip.js b/lib/playlistGenerator/MixFilterClip.js
@@ -87,7 +87,7 @@ MixFilterClip.prototype.doValidate = function mix_doValidate(opts) {
         return false;
     }
 
-    if(that.inner.keyFrameDurations.length > 0) {
+    if(that.inner.keyFrameDurations && that.inner.keyFrameDurations.length > 0) {
         if (that.inner.firstKeyFrameOffset == Infinity) {
             that.logger.warn('that.inner.firstKeyFrameOffset == Infinity && that.inner.keyFrameDurations.length > 0');
             return false;

diff --git a/node_addons/FormatConverter/include/Converter.h b/node_addons/FormatConverter/include/Converter.h
@@ -12,18 +12,42 @@
 
 #include <vector>
 #include <string>
+#include <set>
 #include "Stream.h"
 #include "AVFormat.h"
 
 namespace converter{
 
-
+    class AvLogFilter {
+
+        struct Partial{
+          bool operator ()(const char* l,const char* r) const{
+
+              while(*l && *r){
+                  if( *l != *r )
+                      return *l - *r;
+                  l++;
+                  r++;
+              }
+              return 0;
+        }
+        };
+        std::set<const char*,Partial> m_patterns;
+        std::set<const char*> m_cached;
+    public:
+        void addFilter(const char *p);
+        bool filter(const char * szFmt);
+    };
+
     class ConverterAppInst{
+        AvLogFilter m_filter;
+        static void avlog_cb(void *data, int level, const char * szFmt, va_list varg);
     public:
         ConverterAppInst();
 
         int init(int logLevel = AV_LOG_TRACE);
 
+
         static ConverterAppInst &instance();
 
         const size_t STREAM_BUFFER_SIZE;
@@ -46,17 +70,19 @@ namespace converter{
         COutputCtx  output;
 
         struct ExtraTrackInfo{
-            ExtraTrackInfo(const double &dts)
+            ExtraTrackInfo(const double &dts,std::vector<std::string> wars)
             :lastDTS(AV_NOPTS_VALUE),
             lastPTS(AV_NOPTS_VALUE),
             maxDTS(0),
-            startDTS(dts)
+            startDTS(dts),
+            warnings(wars)
             {}
 
             int64_t lastDTS,
                     lastPTS,
                     maxDTS;
             double startDTS;
+            std::vector<std::string> warnings;
          };
 
         std::vector<ExtraTrackInfo> m_extraTrackInfo;

diff --git a/node_addons/FormatConverter/include/Stream.h b/node_addons/FormatConverter/include/Stream.h
@@ -22,7 +22,7 @@ namespace converter{
         double durationMsec;
         std::vector<double> vecKeyFrameDtsMsec;
         AVMediaType  mtype;
-    };
+     };
 
     struct MediaFileInfo{
 
@@ -33,6 +33,7 @@ namespace converter{
         std::string sig;
         int64_t startTimeUnixMs;
         std::vector<MediaTrackInfo> tracks;
+        std::vector<std::string> vecWarnings;
     };
 
 

diff --git a/node_addons/FormatConverter/setenv.sh b/node_addons/FormatConverter/setenv.sh
@@ -4,6 +4,7 @@
 
 [ "$build_conf" != "Debug" ] && echo "target config: release" ||  echo "target config: debug"
 
+
 os_name=`uname`
 
 function makeFFmpeg()
@@ -34,14 +35,14 @@ function makeFFmpeg()
 
     cd $ffmpegDir
 
-    debug_specifics=""
-    [ "$Release" == "" ] &&  debug_specifics='--enable-debug --disable-optimizations'
+    debug_specifics=
+    [ "$build_conf" = "Debug" ] &&  debug_specifics='--enable-debug --disable-optimizations'
 
     configFileName=$ffmpegDir/lastConfigure
 
 
-    confCmd="./configure --disable-everything --disable-doc --enable-protocol=file --enable-demuxer=mpegts --enable-muxer=rtp_mpegts --enable-parser=h264 --enable-parser=aac --enable-muxer=mp4 --enable-zlib --enable-bsf=aac_adtstoasc --enable-decoder=aac --enable-decoder=h264 --enable-muxer=flv --enable-protocol=rtmp --enable-encoder=libmp3lame"
-    $debug_specifics
+    confCmd="./configure --disable-everything --disable-doc --enable-protocol=file --enable-demuxer=mpegts --enable-muxer=rtp_mpegts --enable-parser=h264 --enable-parser=aac --enable-muxer=mp4 --enable-zlib --enable-bsf=aac_adtstoasc --enable-decoder=aac --enable-decoder=h264 --enable-muxer=flv --enable-protocol=rtmp --enable-encoder=libmp3lame $debug_specifics"
+
 
     [ "$os_name" == "Linux" ] && confCmd="$confCmd --enable-pic"
 

diff --git a/node_addons/FormatConverter/src/Converter.cpp b/node_addons/FormatConverter/src/Converter.cpp
@@ -8,6 +8,7 @@
 
 #include "Converter.h"
 #include <bitset>
+#include <set>
 extern "C"{
 #include <libavformat/movenc.h>
 
@@ -75,8 +76,24 @@ extern "C"{
 };
 
 namespace converter{
+
+    void AvLogFilter::addFilter(const char *s){
+        m_patterns.insert(s);
+    }
+
+    bool AvLogFilter::filter(const char * szFmt) {
+        if(m_patterns.size()){
+            if(m_cached.find(szFmt) != m_cached.end()){
+                return false;
+            }
+            if(m_patterns.find(szFmt) != m_patterns.end()){
+                m_cached.insert(szFmt);
+                return false;
+            }
+        }
+        return true;
+    }
 
-
 
     ConverterAppInst appInst;
 
@@ -93,8 +110,22 @@ namespace converter{
     int ConverterAppInst::init(int logLevel){
         av_log_set_level(logLevel);
         av_register_all();
+        if(logLevel <= AV_LOG_WARNING){
+            m_filter.addFilter("Not writing any edit list");
+            av_log_set_callback(avlog_cb);
+        }
         return 0;
     }
+
+    void ConverterAppInst::avlog_cb(void *data, int level, const char * szFmt, va_list varg){
+        if(level > av_log_get_level()){
+            return;
+        }
+        if(!ConverterAppInst::instance().m_filter.filter(szFmt)){
+            return;
+        }
+        av_log_default_callback(data,level,szFmt,varg);
+    }
 
 
     Converter::Converter()
@@ -182,6 +213,8 @@ namespace converter{
 
         for( size_t i = 0 , output_stream = 0; i < input->nb_streams; i++){
 
+            std::vector<std::string> errors;
+
             AVStream *in_stream =input->streams[i];
 
             int64_t stmStartTimeMs = av_rescale(in_stream->start_time,1000 * in_stream->time_base.num,in_stream->time_base.den);
@@ -198,7 +231,7 @@ namespace converter{
 
             m_streamMapper[i] = output_stream++;
 
-            m_extraTrackInfo.push_back(ExtraTrackInfo(dts2msec(getStreamStartTime(in_stream),in_stream->time_base)));
+            m_extraTrackInfo.push_back(ExtraTrackInfo(dts2msec(getStreamStartTime(in_stream),in_stream->time_base),errors));
 
             _S(avcodec_copy_context(out_stream->codec, in_stream->codec));
 
@@ -220,7 +253,11 @@ namespace converter{
             }
         }
 
-        _S(avformat_write_header(*output, nullptr));
+         AVDictionary *opts = nullptr;
+        _S(av_dict_set(&opts, "use_editlist", "0", 0));
+        std::unique_ptr<AVDictionary> optsptr(opts);
+        _S(avformat_write_header(*output, &opts));
+        optsptr.release();
 
         state = PUSHING;
 

diff --git a/node_addons/FormatConverter/src/NodeStream.cpp b/node_addons/FormatConverter/src/NodeStream.cpp
@@ -175,6 +175,7 @@ namespace converter {
                     trackInfo->Set(String::NewFromUtf8(isolate, "firstEncoderDTS"),Number::New(isolate, iter->firstEncoderDtsMsec));
                     trackInfo->Set(String::NewFromUtf8(isolate, "wrapEncoderDTS"),Number::New(isolate, iter->wrapEncoderDtsMsec));
                     trackInfo->Set(String::NewFromUtf8(isolate, "keyFrameDTS"),vecKeyFrameDtsMsec);
+
                 }
                     break;
                 default:

diff --git a/node_addons/FormatConverter/test/mp4writerTest.js b/node_addons/FormatConverter/test/mp4writerTest.js
@@ -5,15 +5,10 @@ var Url = require('url');
 var _ = require('underscore');
 var util = require('util');
 
-var tsFilePath = '/Users/igors/media-uh6sbv3i0_251.ts';//"/Users/igors/dvr/dvrContentRootPath/1_abc123/1/media-ul0o1lom6_w1600782441_670.ts.mp4_saved.ts";//__dirname+'/../resources/media-uixh2a1qh_w1892051821_472.ts';
+var tsFilePath = '/Users/igors/media-u3f95g2zn_21849.ts';//"/Users/igors/dvr/dvrContentRootPath/1_abc123/1/media-ul0o1lom6_w1600782441_670.ts.mp4_saved.ts";//__dirname+'/../resources/media-uixh2a1qh_w1892051821_472.ts';
 var httpPath = 'http://localhost/wn/media-uhe4wm3o6_b475136_144354218.ts';
 console.log("tsFilePath=",tsFilePath);
-var ts2mp4 = new MP4WRITER.MP4WriteStream(tsFilePath + '.mp4', {
-        debug: console.log,
-        info: console.log,
-        warn: console.log,
-        error: console.log
-    });
+var ts2mp4 = new MP4WRITER.MP4WriteStream(tsFilePath + '.mp4', "");
 
 var origUrl = {
     'http:': function (url, cb) {