Skip to content

Commit

Permalink
1.0 file tree refactor (#260)
Browse files Browse the repository at this point in the history 
* -1.0-file-tree-refactor

* - chunk name refactor - shorten file name

* -refactor

* - fix regex

* -1.0-bad_key_frame_timestamps
  • Loading branch information
@igorshevach
igorshevach authored and gadyaari committed Sep 21, 2016
1 parent 541f379 commit 2a2e29b
Show file tree
Hide file tree
Showing 6 changed files with 75 additions and 58 deletions.
14 changes: 9 additions & 5 deletions common/PersistenceFormat.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ var qio = require('q-io/fs');
var _ = require('underscore');
var Q = require('q');

const tsChunktMatch = new RegExp(/media-([^_]+).*(_[\d]+.ts).*/);
const tsChunktMatch = new RegExp(/media-([^_]+).*?([\d]+)\.ts.*/);

module.exports = {
module.exports = persistenceFormat = {

getEntryBasePath: function (entryId) {
return path.join(config.get('rootFolderPath'), entryId);
Expand All @@ -20,6 +20,10 @@ module.exports = {
return path.dirname(path.dirname(directory));
},

getRelativePathFromFull: function (fullPath) {
return fullPath.substr(persistenceFormat.getBasePathFromFull(fullPath).length);
},

getFlavorFullPath: function (entryId, flavorName) {
return path.join(config.get('rootFolderPath'), entryId, flavorName.toString());
},
Expand All @@ -36,7 +40,7 @@ module.exports = {
getTSChunknameFromMP4FileName: function(mp4FileName){
return mp4FileName.replace('.mp4','.ts');
},

createHierarchyPath: function(destPath, lastFileHash) {
let hash = new Date().getHours().toString();
let fileFullPath = path.join(destPath, (hash) < 10 ? ("0" + hash) : hash);
Expand All @@ -51,10 +55,10 @@ module.exports = {
});
},

normalizeChunkName: function(tsChunkName){
compressChunkName: function(tsChunkName){
var matched = tsChunktMatch.exec( tsChunkName );
if(matched){
return matched[1] + matched[2];
return matched[1] + '-' + matched[2] + '.mp4';
}
return tsChunkName;
}
Expand Down
2 changes: 1 addition & 1 deletion lib/MP4WriteStream.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ var MP4WriteStream = (function(){
var that = this;

that.logger = loggerModule.getLogger("MP4WriteStream", loggerInfo);
that.tsChunkPath = chunkPath;
that.tsChunkPath = persistenceFormat.getTSChunknameFromMP4FileName(chunkPath);

that.savedChunk = config.get('preserveOriginalChunk') ? fs.createWriteStream(chunkPath) : undefined;

Expand Down
60 changes: 27 additions & 33 deletions lib/entry/FlavorDownloader.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,26 +117,25 @@ var FlavorDownloader = (function() {
});
};

var downloadChunks = function(baseUrl, chunksToDownload, downloadedM3u8) {
var downloadChunks = function(baseUrl, chunks, m3u8) {
var that = this;
return Q.allSettled(_.map(chunksToDownload, function(chunkName) {
var chunkUrl = url.resolve(baseUrl + '/', chunkName);
return Q.allSettled(_.map(chunks, function(chunkName) {
var chunkUrl = url.resolve(baseUrl + '/', that.chunksToDownload[chunkName]);

return persistenceFormat.createHierarchyPath(that.destFolderPath, that.lastFilePathHash)
.then(function({fileFullPath, hash}) {
that.lastFilePathHash = hash;
//return the promise of the downloaded chunk
return that.tsHttpUtils.downloadConvert(chunkUrl, path.join(fileFullPath, chunkName))
.then(function(mp4File) {
mp4File.flavor = that.flavor;
mp4File.chunkName = chunkName;
if(downloadedM3u8.properties.targetDuration) {
mp4File.targetDuration = downloadedM3u8.properties.targetDuration * 1000;
mp4File.windowSize = downloadedM3u8.items.PlaylistItem.length * mp4File.targetDuration * 2;
mp4File.chunkName = path.basename(mp4File.path);
if(m3u8.properties.targetDuration) {
mp4File.targetDuration = m3u8.properties.targetDuration * 1000;
mp4File.windowSize = m3u8.items.PlaylistItem.length * mp4File.targetDuration * 2;
}
delete that.chunksToDownload[chunkName];
that.downloadedChunks[chunkName] = true;
//create link between original ts chunk name and mp4 file name
that.downloadedChunks[path.basename(mp4File.path)] = chunkName;
return mp4File;
})
.catch(function(err) {
Expand Down Expand Up @@ -180,18 +179,22 @@ var FlavorDownloader = (function() {
};
var updateListOfNewChunksToProcess = function(downloadedM3u8) {
var newChunks = _.chain(downloadedM3u8.items.PlaylistItem)
.map(function(item){
return item.get('uri');
.map((item)=>{
var uri = item.get('uri');
return {
uri: uri,
cmpURI: persistenceFormat.compressChunkName(uri)
};
})
.filter(function(newChunk) {
return !that.downloadedChunks[newChunk] && !that.chunksToDownload[newChunk];
.filter((chunk) => {
return !that.downloadedChunks[chunk.cmpURI] && !that.chunksToDownload[chunk.cmpURI];
})
.value();

// update that these new chunks need to be downloaded
_.each(newChunks, function (newChunk) {
that.logger.debug("Adding %s to the list of chunks to download", newChunk);
that.chunksToDownload[newChunk] = true;
that.logger.debug("Adding %j to the list of chunks to download", newChunk);
that.chunksToDownload[newChunk.cmpURI] = newChunk.uri;
});

if (Object.keys(that.chunksToDownload).length >= maxAmountOfDownloadPendingChunks) {
Expand All @@ -214,22 +217,15 @@ var FlavorDownloader = (function() {
// (as they will never be processed)
_.each(manifestUpdatePromiseResult.value.removedChunks, function (chunkName) {

var realChunkName = that.downloadedChunks[chunkName];

if(typeof realChunkName === 'string'){
// delete link to original chunkName
if(that.downloadedChunks[chunkName]) {
delete that.downloadedChunks[chunkName];
chunkName = realChunkName;
}

if (that.chunksToDownload[chunkName]) {
that.logger.error("Removing chunk " + chunkName + " from chunksToDownload - this shouldn't happen");
delete that.chunksToDownload[chunkName];
if (that.chunksToDownload[chunkName]) {
that.logger.error("Removing chunk " + chunkName + " from chunksToDownload - this shouldn't happen");
delete that.chunksToDownload[chunkName];
}
that.logger.debug("Delete %s from the list of downloaded chunks", chunkName);
}

that.logger.debug("Delete %s from the list of downloaded chunks", chunkName);
delete that.downloadedChunks[chunkName];

});
}
};
Expand Down Expand Up @@ -263,7 +259,7 @@ var FlavorDownloader = (function() {
})
.then(function(m3u) {
// Download chunks - all the chunks that were pending download beforehand along with the new chunks
return downloadChunks.call(that, baseUrl, Object.keys(that.chunksToDownload),m3u);
return downloadChunks.call(that, baseUrl, _.keys(that.chunksToDownload), m3u);
})
.then(function(mp4Files){
// generate updated playlist
Expand Down Expand Up @@ -314,18 +310,16 @@ var FlavorDownloader = (function() {
var checkCrashedFiles = [];
_.each(downloadedChunks, function (c) {
var fileName = path.basename(c);
var tsFileName = persistenceFormat.getTSChunknameFromMP4FileName(fileName);
if (that.playlistGenerator.checkFileExists(fileName)) {
that.logger.trace("Adding %s to downloadedChunks", fileName);
that.downloadedChunks[tsFileName] = true;
that.downloadedChunks[fileName] = tsFileName;
that.downloadedChunks[fileName] = true;
} else {
// test against 0 length file - guard against endless looping caused
// by crashing while attempting to convert chunk
checkCrashedFiles.push(qfs.stat(c).then(function (stats) {
if (!stats.size) {
that.logger.warn("Zero size file! Adding %s to chunksToDownload", fileName);
that.chunksToDownload[tsFileName] = true;
that.downloadedChunks[fileName] = true;
}
}));
}
Expand Down
54 changes: 36 additions & 18 deletions lib/playlistGenerator/PlaylistGenerator.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ function PlaylistGenerator (entryObject) {
that.startPromise = null;
that.persistPromise = qResolved;
that.playListLimits = {
manifestTimeWindowInMsec: Math.ceil(entryObject.playWindow * 1000 * hlsStoreFilefactor),
manifestTimeWindowInMsec:Math.ceil(entryObject.playWindow * 1000 * hlsStoreFilefactor),
maxClipCountPerPlaylist:Math.max(10,entryObject.maxChunkCount)
};
that.playlistPath = path.join(persistenceFormat.getEntryBasePath(that.entryId),persistenceFormat.getMasterManifestName());
Expand Down Expand Up @@ -87,19 +87,31 @@ var isRunning = function isRunning(){
return this.startPromise && !this.stopPromise;
};

var checkDelayedRemovalChunksReady = function(removedChunks){
function getDelayedForFlavor (flavor) {

var seq = this.playlistImp.getSequenceForFlavor(flavor);

if(!seq.scheduledForRemoval){
seq.scheduledForRemoval = {};
}

return seq.scheduledForRemoval;
}

var checkDelayedRemovalChunksReady = function(flavor,removedChunks){
var that = this;

var delayedChunks = getDelayedForFlavor.call(that,flavor);

var n = that.playlistImp.getCurrentTime();
var removedChunks = removedChunks || [];
var keys = _.keys(that.scheduledForRemoval);
_.every(_.values(that.scheduledForRemoval),function(fi,index) {
var keys = _.keys(delayedChunks);
_.every(_.values(delayedChunks),function(fi,index) {
if (n > fi.when) {
var ready = keys[index];
that.logger.warn("PlaylistGenerator.delayChunkRemoval. chunk is ready %j",ready);
removeFile.call(that,fi.path);
removedChunks.push(ready);
delete that.scheduledForRemoval[ready];
removedChunks.push(fi.path);
delete delayedChunks[ready];
return true;
} else {
return false;
Expand All @@ -108,29 +120,27 @@ var checkDelayedRemovalChunksReady = function(removedChunks){
return removedChunks;
};

var handleClipError = function(fileInfo){
function handleClipError (fileInfo){
var that = this;

if(! that.scheduledForRemoval){
that.scheduledForRemoval = {};
}
var delayedChunks = getDelayedForFlavor.call(that,fileInfo.flavor);

var chunkname = path.basename(fileInfo.path);

if(that.scheduledForRemoval[chunkname]) {
if(delayedChunks[chunkname]) {
return;
}

var keys = _.keys(that.scheduledForRemoval);
var keys = _.keys(delayedChunks);

var n = that.playlistImp.getCurrentTime();

var when = n + Math.max(fileInfo.windowSize || 0,1000*60*5);
var when = n;// + Math.max(fileInfo.windowSize || 0,1000*60*5);

that.logger.warn("handleClipError. chunk %j. will be removed in %j ms",chunkname,when-n);

that.scheduledForRemoval[chunkname] = { when: when, path: fileInfo.path };
};
delayedChunks[chunkname] = { when: when, path: persistenceFormat.getRelativePathFromFull(fileInfo.path) };
}

// private insert a new media chunk
var addNewClip = function addNewClip (fileInfo){
Expand Down Expand Up @@ -172,7 +182,7 @@ var removeFile = function (pathToDelete) {
break;
}
}
logOp.call(that.logger,'removeFile. Error deleting a chunk %s', ErrorUtils.error2string(err));
logOp.call(that.logger,'removeFile. Error deleting a chunk %s', ErrorUtils.error2string(err));
});
};

Expand All @@ -184,7 +194,7 @@ var checkRemoveChunks = function(flavor){

var removedChunks = seq.checkExpires(that.playlistImp.getCurrentTime(), that.playlistImp.totalDuration);

checkDelayedRemovalChunksReady.call(that,removedChunks);
checkDelayedRemovalChunksReady.call(that,flavor,removedChunks);

if(removedChunks.length) {
var flavorBasePath = persistenceFormat.getFlavorFullPath(that.entryId,flavor);
Expand Down Expand Up @@ -439,6 +449,14 @@ PlaylistGenerator.prototype.validate = function() {
};

PlaylistGenerator.prototype.checkFileExists = function(fileName){
var that = this;

var delayedChunks = getDelayedForFlavor.call(that,flavor);

if( delayedChunks[path.basename(fileName)] ){
return true;
}

return this.playlistImp.checkFileExists(fileName);
};

Expand Down
1 change: 1 addition & 0 deletions lib/utils/http-utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ var HttpDownloader = function(loggerInfo ,compareFn) {
if (err) {
onError(err);
} else {
downloadedTime = downloadedTime || new Date();
response
.pipe(stream)
.on('error', onError)
Expand Down
2 changes: 1 addition & 1 deletion node_addons/FormatConverter/test/mp4writerTest.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ var util = require('util');
var tsFilePath = process.argv[2];//'/Users/igors/media-u4cc7m30h_b1496000_3383.ts';//"/Users/igors/dvr/dvrContentRootPath/1_abc123/1/media-ul0o1lom6_w1600782441_670.ts.mp4_saved.ts";//__dirname+'/../resources/media-uixh2a1qh_w1892051821_472.ts';
var httpPath = 'http://localhost/wn/media-uhe4wm3o6_b475136_144354218.ts';
console.log("tsFilePath=",tsFilePath);
var ts2mp4 = new MP4WRITER.MP4WriteStream(tsFilePath + '.mp4', "");
var ts2mp4 = new MP4WRITER.MP4WriteStream(tsFilePath, "");

var origUrl = {
'http:': function (url, cb) {
Expand Down

0 comments on commit 2a2e29b

Please sign in to comment.