gulpfile.js

/*eslint-env node*/
'use strict';

var fs = require('fs');
var path = require('path');
var os = require('os');
var child_process = require('child_process');
var crypto = require('crypto');
var zlib = require('zlib');
var readline = require('readline');
var request = require('request');

var globby = require('globby');
var gulpTap = require('gulp-tap');
var gulpUglify = require('gulp-uglify');
var open = require('open');
var rimraf = require('rimraf');
var glslStripComments = require('glsl-strip-comments');
var mkdirp = require('mkdirp');
var mergeStream = require('merge-stream');
var streamToPromise = require('stream-to-promise');
var gulp = require('gulp');
var gulpInsert = require('gulp-insert');
var gulpZip = require('gulp-zip');
var gulpRename = require('gulp-rename');
var gulpReplace = require('gulp-replace');
var Promise = require('bluebird');
var Karma = require('karma');
var yargs = require('yargs');
var AWS = require('aws-sdk');
var mime = require('mime');
var rollup = require('rollup');
var rollupPluginStripPragma = require('rollup-plugin-strip-pragma');
var rollupPluginExternalGlobals = require('rollup-plugin-external-globals');
var rollupPluginUglify = require('rollup-plugin-uglify');
var cleanCSS = require('gulp-clean-css');

var packageJson = require('./package.json');
var version = packageJson.version;
if (/\.0$/.test(version)) {
    version = version.substring(0, version.length - 2);
}

var karmaConfigFile = path.join(__dirname, 'Specs/karma.conf.js');
var travisDeployUrl = 'http://cesium-dev.s3-website-us-east-1.amazonaws.com/cesium/';

//Gulp doesn't seem to have a way to get the currently running tasks for setting
//per-task variables.  We use the command line argument here to detect which task is being run.
var taskName = process.argv[2];
var noDevelopmentGallery = taskName === 'release' || taskName === 'makeZipFile';
var minifyShaders = taskName === 'minify' || taskName === 'minifyRelease' || taskName === 'release' || taskName === 'makeZipFile' || taskName === 'buildApps';

var verbose = yargs.argv.verbose;

var concurrency = yargs.argv.concurrency;
if (!concurrency) {
    concurrency = os.cpus().length;
}

var sourceFiles = ['Source/**/*.js',
                   '!Source/*.js',
                   '!Source/Workers/**',
                   '!Source/WorkersES6/**',
                   'Source/WorkersES6/createTaskProcessorWorker.js',
                   '!Source/ThirdParty/Workers/**',
                   '!Source/ThirdParty/google-earth-dbroot-parser.js',
                   '!Source/ThirdParty/pako_inflate.js',
                   '!Source/ThirdParty/crunch.js'];

var watchedFiles = ['Source/**/*.js',
                    '!Source/Cesium.js',
                    '!Source/Build/**',
                    '!Source/Shaders/**/*.js',
                    'Source/Shaders/**/*.glsl',
                    '!Source/ThirdParty/Shaders/*.js',
                    'Source/ThirdParty/Shaders/*.glsl',
                    '!Source/Workers/**',
                    'Source/Workers/cesiumWorkerBootstrapper.js',
                    'Source/Workers/transferTypedArrayTest.js',
                    '!Specs/SpecList.js'];

var filesToClean = ['Source/Cesium.js',
                    'Source/Shaders/**/*.js',
                    'Source/Workers/**',
                    '!Source/Workers/cesiumWorkerBootstrapper.js',
                    '!Source/Workers/transferTypedArrayTest.js',
                    'Source/ThirdParty/Shaders/*.js',
                    'Specs/SpecList.js',
                    'Apps/Sandcastle/jsHintOptions.js',
                    'Apps/Sandcastle/gallery/gallery-index.js',
                    'Apps/Sandcastle/templates/bucket.css',
                    'Cesium-*.zip',
                    'cesium-*.tgz'];

var filesToConvertES6 = ['Source/**/*.js',
                         'Specs/**/*.js',
                         '!Source/ThirdParty/**',
                         '!Source/Cesium.js',
                         '!Source/copyrightHeader.js',
                         '!Source/Shaders/**',
                         '!Source/Workers/cesiumWorkerBootstrapper.js',
                         '!Source/Workers/transferTypedArrayTest.js',
                         '!Specs/karma-main.js',
                         '!Specs/karma.conf.js',
                         '!Specs/spec-main.js',
                         '!Specs/SpecList.js',
                         '!Specs/TestWorkers/**'
                        ];

function rollupWarning(message) {
    // Ignore eval warnings in third-party code we don't have control over
    if (message.code === 'EVAL' && /(protobuf-minimal|crunch)\.js$/.test(message.loc.file)) {
        return;
    }
    console.log(message);
}

function createWorkers() {
    rimraf.sync('Build/createWorkers');

    globby.sync([
        'Source/Workers/**',
        '!Source/Workers/cesiumWorkerBootstrapper.js',
        '!Source/Workers/transferTypedArrayTest.js'
    ]).forEach(function(file) {
        rimraf.sync(file);
    });

    var workers = globby.sync([
        'Source/WorkersES6/**'
    ]);

    return rollup.rollup({
        input: workers,
        onwarn: rollupWarning
    }).then(function(bundle) {
        return bundle.write({
            dir: 'Build/createWorkers',
            banner: '/* This file is automatically rebuilt by the Cesium build process. */',
            format: 'amd'
        });
    }).then(function(){
        return streamToPromise(
            gulp.src('Build/createWorkers/**').pipe(gulp.dest('Source/Workers'))
        );
    }).then(function() {
        rimraf.sync('Build/createWorkers');
    });
}

gulp.task('build', function() {
    mkdirp.sync('Build');
    glslToJavaScript(minifyShaders, 'Build/minifyShaders.state');
    createCesiumJs();
    createSpecList();
    createJsHintOptions();
    return Promise.join(createWorkers(), createGalleryList());
});

gulp.task('build-watch', function() {
    return gulp.watch(watchedFiles, gulp.series('build'));
});

gulp.task('buildApps', function() {
    return Promise.join(
        buildCesiumViewer(),
        buildSandcastle()
    );
});

gulp.task('build-specs', function buildSpecs() {
    var externalCesium = rollupPluginExternalGlobals({
        '../Source/Cesium.js': 'Cesium',
        '../../Source/Cesium.js': 'Cesium',
        '../../../Source/Cesium.js': 'Cesium',
        '../../../../Source/Cesium.js': 'Cesium'
    });

    var removePragmas = rollupPluginStripPragma({
        pragmas: ['debug']
    });

    var promise = Promise.join(
        rollup.rollup({
            input: 'Specs/SpecList.js',
            plugins: [externalCesium],
            onwarn: rollupWarning
        }).then(function(bundle) {
            return bundle.write({
                file: 'Build/Specs/Specs.js',
                format: 'iife'
            });
        }).then(function(){
            return rollup.rollup({
                input: 'Specs/spec-main.js',
                plugins: [removePragmas, externalCesium]
            }).then(function(bundle) {
                return bundle.write({
                    file: 'Build/Specs/spec-main.js',
                    format: 'iife'
                });
            });
        }).then(function(){
            return rollup.rollup({
                input: 'Specs/karma-main.js',
                plugins: [removePragmas, externalCesium],
                onwarn: rollupWarning
            }).then(function(bundle) {
                return bundle.write({
                    file: 'Build/Specs/karma-main.js',
                    name: 'karmaMain',
                    format: 'iife'
                });
            });
        })
    );

    return promise;
});

gulp.task('clean', function(done) {
    rimraf.sync('Build');
    globby.sync(filesToClean).forEach(function(file) {
        rimraf.sync(file);
    });
    done();
});

function cloc() {
    var cmdLine;
    var clocPath = path.join('node_modules', 'cloc', 'lib', 'cloc');

    //Run cloc on primary Source files only
    var source = new Promise(function(resolve, reject) {
        cmdLine = 'perl ' + clocPath + ' --quiet --progress-rate=0' +
                  ' Source/ --exclude-dir=Assets,ThirdParty,Workers --not-match-f=copyrightHeader.js';

        child_process.exec(cmdLine, function(error, stdout, stderr) {
            if (error) {
                console.log(stderr);
                return reject(error);
            }
            console.log('Source:');
            console.log(stdout);
            resolve();
        });
    });

    //If running cloc on source succeeded, also run it on the tests.
    return source.then(function() {
        return new Promise(function(resolve, reject) {
            cmdLine = 'perl ' + clocPath + ' --quiet --progress-rate=0' +
                      ' Specs/ --exclude-dir=Data';
            child_process.exec(cmdLine, function(error, stdout, stderr) {
                if (error) {
                    console.log(stderr);
                    return reject(error);
                }
                console.log('Specs:');
                console.log(stdout);
                resolve();
            });
        });
    });
}

gulp.task('cloc', gulp.series('clean', cloc));

function combine() {
    var outputDirectory = path.join('Build', 'CesiumUnminified');
    return combineJavaScript({
        removePragmas: false,
        optimizer: 'none',
        outputDirectory: outputDirectory
    });
}

gulp.task('combine', gulp.series('build', combine));
gulp.task('default', gulp.series('combine'));

function combineRelease() {
    var outputDirectory = path.join('Build', 'CesiumUnminified');
    return combineJavaScript({
        removePragmas: true,
        optimizer: 'none',
        outputDirectory: outputDirectory
    });
}

gulp.task('combineRelease', gulp.series('build', combineRelease));

//Builds the documentation
function generateDocumentation() {
    var envPathSeperator = os.platform() === 'win32' ? ';' : ':';

    return new Promise(function(resolve, reject) {
        child_process.exec('jsdoc --configure Tools/jsdoc/conf.json', {
            env : {
                PATH : process.env.PATH + envPathSeperator + 'node_modules/.bin',
                CESIUM_VERSION : version
            }
        }, function(error, stdout, stderr) {
            if (error) {
                console.log(stderr);
                return reject(error);
            }
            console.log(stdout);
            var stream = gulp.src('Documentation/Images/**').pipe(gulp.dest('Build/Documentation/Images'));
            return streamToPromise(stream).then(resolve);
        });
    });
}
gulp.task('generateDocumentation', generateDocumentation);

gulp.task('generateDocumentation-watch', function() {
    return generateDocumentation().done(function() {
        console.log('Listening for changes in documentation...');
        return gulp.watch(sourceFiles, gulp.series('generateDocumentation'));
    });
});

gulp.task('release', gulp.series('build', combine, minifyRelease, generateDocumentation));

gulp.task('makeZipFile', gulp.series('release', function() {
    //For now we regenerate the JS glsl to force it to be unminified in the release zip
    //See https://github.com/CesiumGS/cesium/pull/3106#discussion_r42793558 for discussion.
    glslToJavaScript(false, 'Build/minifyShaders.state');

    var builtSrc = gulp.src([
        'Build/Cesium/**',
        'Build/CesiumUnminified/**',
        'Build/Documentation/**'
    ], {
        base : '.'
    });

    var staticSrc = gulp.src([
        'Apps/**',
        '!Apps/Sandcastle/gallery/development/**',
        'Source/**',
        'Specs/**',
        'ThirdParty/**',
        'favicon.ico',
        'gulpfile.js',
        'server.js',
        'package.json',
        'LICENSE.md',
        'CHANGES.md',
        'README.md',
        'web.config'
    ], {
        base : '.'
    });

    var indexSrc = gulp.src('index.release.html').pipe(gulpRename('index.html'));

    return mergeStream(builtSrc, staticSrc, indexSrc)
        .pipe(gulpTap(function(file) {
            // Work around an issue with gulp-zip where archives generated on Windows do
            // not properly have their directory executable mode set.
            // see https://github.com/sindresorhus/gulp-zip/issues/64#issuecomment-205324031
            if (file.isDirectory()) {
                file.stat.mode = parseInt('40777', 8);
            }
        }))
        .pipe(gulpZip('Cesium-' + version + '.zip'))
        .pipe(gulp.dest('.'));
}));

gulp.task('minify', gulp.series('build', function() {
    return combineJavaScript({
        removePragmas : false,
        optimizer : 'uglify2',
        outputDirectory : path.join('Build', 'Cesium')
    });
}));

function minifyRelease() {
    return combineJavaScript({
        removePragmas: true,
        optimizer: 'uglify2',
        outputDirectory: path.join('Build', 'Cesium')
    });
}

gulp.task('minifyRelease', gulp.series('build', minifyRelease));

function isTravisPullRequest() {
    return process.env.TRAVIS_PULL_REQUEST !== undefined && process.env.TRAVIS_PULL_REQUEST !== 'false';
}

gulp.task('deploy-s3', function(done) {
    if (isTravisPullRequest()) {
        console.log('Skipping deployment for non-pull request.');
        done();
        return;
    }

    var argv = yargs.usage('Usage: deploy-s3 -b [Bucket Name] -d [Upload Directory]')
        .demand(['b', 'd']).argv;

    var uploadDirectory = argv.d;
    var bucketName = argv.b;
    var cacheControl = argv.c ? argv.c : 'max-age=3600';

    if (argv.confirm) {
        // skip prompt for travis
        deployCesium(bucketName, uploadDirectory, cacheControl, done);
        return;
    }

    var iface = readline.createInterface({
        input: process.stdin,
        output: process.stdout
    });

    // prompt for confirmation
    iface.question('Files from your computer will be published to the ' + bucketName + ' bucket. Continue? [y/n] ', function(answer) {
        iface.close();
        if (answer === 'y') {
            deployCesium(bucketName, uploadDirectory, cacheControl, done);
        } else {
            console.log('Deploy aborted by user.');
            done();
        }
    });

});

// Deploy cesium to s3
function deployCesium(bucketName, uploadDirectory, cacheControl, done) {
    var readFile = Promise.promisify(fs.readFile);
    var gzip = Promise.promisify(zlib.gzip);
    var concurrencyLimit = 2000;

    var s3 = new AWS.S3({
        maxRetries : 10,
        retryDelayOptions : {
            base : 500
        }
    });

    var existingBlobs = [];
    var totalFiles = 0;
    var uploaded = 0;
    var skipped = 0;
    var errors = [];

    var prefix = uploadDirectory + '/';
    return listAll(s3, bucketName, prefix, existingBlobs)
        .then(function() {
            return globby([
                'Apps/**',
                'Build/**',
                'Source/**',
                'Specs/**',
                'ThirdParty/**',
                '*.md',
                'favicon.ico',
                'gulpfile.js',
                'index.html',
                'package.json',
                'server.js',
                'web.config',
                '*.zip',
                '*.tgz'
            ], {
                dot : true // include hidden files
            });
        }).then(function(files) {
            return Promise.map(files, function(file) {
                var blobName = uploadDirectory + '/' + file;
                var mimeLookup = getMimeType(blobName);
                var contentType = mimeLookup.type;
                var compress = mimeLookup.compress;
                var contentEncoding = compress ? 'gzip' : undefined;
                var etag;

                totalFiles++;

                return readFile(file)
                .then(function(content) {
                    if (!compress) {
                        return content;
                    }

                    var alreadyCompressed = (content[0] === 0x1f) && (content[1] === 0x8b);
                    if (alreadyCompressed) {
                        console.log('Skipping compressing already compressed file: ' + file);
                        return content;
                    }

                    return gzip(content);
                })
                .then(function(content) {
                    // compute hash and etag
                    var hash = crypto.createHash('md5').update(content).digest('hex');
                    etag = crypto.createHash('md5').update(content).digest('base64');

                    var index = existingBlobs.indexOf(blobName);
                    if (index <= -1) {
                        return content;
                    }

                    // remove files as we find them on disk
                    existingBlobs.splice(index, 1);

                    // get file info
                    return s3.headObject({
                            Bucket: bucketName,
                            Key: blobName
                        }).promise().then(function(data) {
                            if (data.ETag !== ('"' + hash + '"') ||
                                data.CacheControl !== cacheControl ||
                                data.ContentType !== contentType ||
                                data.ContentEncoding !== contentEncoding) {
                                return content;
                            }

                            // We don't need to upload this file again
                            skipped++;
                            return undefined;
                        })
                        .catch(function(error) {
                            errors.push(error);
                        });
                })
                .then(function(content) {
                    if (!content) {
                        return;
                    }

                    if (verbose) {
                        console.log('Uploading ' + blobName + '...');
                    }
                    var params = {
                        Bucket : bucketName,
                        Key : blobName,
                        Body : content,
                        ContentMD5 : etag,
                        ContentType : contentType,
                        ContentEncoding : contentEncoding,
                        CacheControl : cacheControl
                    };

                    return s3.putObject(params).promise()
                        .then(function() {
                            uploaded++;
                        })
                        .catch(function(error) {
                            errors.push(error);
                        });
                });
            }, {concurrency : concurrencyLimit});
        }).then(function() {
            console.log('Skipped ' + skipped + ' files and successfully uploaded ' + uploaded + ' files of ' + (totalFiles - skipped) + ' files.');
            if (existingBlobs.length === 0) {
                return;
            }

            var objectsToDelete = [];
            existingBlobs.forEach(function(file) {
                //Don't delete generate zip files.
                if (!/\.(zip|tgz)$/.test(file)) {
                    objectsToDelete.push({Key : file});
                }
            });

            if (objectsToDelete.length > 0) {
                console.log('Cleaning ' + objectsToDelete.length + ' files...');

                // If more than 1000 files, we must issue multiple requests
                var batches = [];
                while (objectsToDelete.length > 1000) {
                    batches.push(objectsToDelete.splice(0, 1000));
                }
                batches.push(objectsToDelete);

                return Promise.map(batches, function(objects) {
                    return s3.deleteObjects({
                        Bucket: bucketName,
                        Delete: {
                            Objects: objects
                        }
                    }).promise().then(function() {
                        if (verbose) {
                            console.log('Cleaned ' + objects.length + ' files.');
                        }
                    });
                }, {concurrency : concurrency});
            }
        }).catch(function(error) {
            errors.push(error);
        }).then(function() {
            if (errors.length === 0) {
                done();
                return;
            }

            console.log('Errors: ');
            errors.map(function(e) {
                console.log(e);
            });
            done(1);
        });
}

function getMimeType(filename) {
    var mimeType = mime.getType(filename);
    if (mimeType) {
        //Compress everything except zipfiles, binary images, and video
        var compress = !/^(image\/|video\/|application\/zip|application\/gzip)/i.test(mimeType);
        if (mimeType === 'image/svg+xml') {
            compress = true;
        }
        return { type: mimeType, compress: compress };
    }

    //Non-standard mime types not handled by mime
    if (/\.(glsl|LICENSE|config|state)$/i.test(filename)) {
        return { type: 'text/plain', compress: true };
    } else if (/\.(czml|topojson)$/i.test(filename)) {
        return { type: 'application/json', compress: true };
    } else if (/\.(crn|tgz)$/i.test(filename)) {
        return { type: 'application/octet-stream', compress: false };
    }

    // Handle dotfiles, such as .jshintrc
    var baseName = path.basename(filename);
    if (baseName[0] === '.' || baseName.indexOf('.') === -1) {
        return { type: 'text/plain', compress: true };
    }

    // Everything else can be octet-stream compressed but print a warning
    // if we introduce a type we aren't specifically handling.
    if (!/\.(terrain|b3dm|geom|pnts|vctr|cmpt|i3dm|metadata)$/i.test(filename)) {
        console.log('Unknown mime type for ' + filename);
    }

    return { type: 'application/octet-stream', compress: true };
}

// get all files currently in bucket asynchronously
function listAll(s3, bucketName, prefix, files, marker) {
    return s3.listObjects({
        Bucket: bucketName,
        MaxKeys: 1000,
        Prefix: prefix,
        Marker: marker
    }).promise().then(function(data) {
        var items = data.Contents;
        for (var i = 0; i < items.length; i++) {
            files.push(items[i].Key);
        }

        if (data.IsTruncated) {
            // get next page of results
            return listAll(s3, bucketName, prefix, files, files[files.length - 1]);
        }
    });
}

gulp.task('deploy-set-version', function(done) {
    var buildVersion = yargs.argv.buildVersion;
    if (buildVersion) {
        // NPM versions can only contain alphanumeric and hyphen characters
        packageJson.version += '-' + buildVersion.replace(/[^[0-9A-Za-z-]/g, '');
        fs.writeFileSync('package.json', JSON.stringify(packageJson, undefined, 2));
    }
    done();
});

gulp.task('deploy-status', function() {
    if (isTravisPullRequest()) {
        console.log('Skipping deployment status for non-pull request.');
        return Promise.resolve();
    }

    var status = yargs.argv.status;
    var message = yargs.argv.message;

    var deployUrl = travisDeployUrl + process.env.TRAVIS_BRANCH + '/';
    var zipUrl = deployUrl + 'Cesium-' + packageJson.version + '.zip';
    var npmUrl = deployUrl + 'cesium-' + packageJson.version + '.tgz';
    var coverageUrl = travisDeployUrl + process.env.TRAVIS_BRANCH + '/Build/Coverage/index.html';

    return Promise.join(
        setStatus(status, deployUrl, message, 'deployment'),
        setStatus(status, zipUrl, message, 'zip file'),
        setStatus(status, npmUrl, message, 'npm package'),
        setStatus(status, coverageUrl, message, 'coverage results')
    );
});

function setStatus(state, targetUrl, description, context) {
    // skip if the environment does not have the token
    if (!process.env.TOKEN) {
        return;
    }

    var requestPost = Promise.promisify(request.post);
    return requestPost({
         url: 'https://api.github.com/repos/' + process.env.TRAVIS_REPO_SLUG + '/statuses/' + process.env.TRAVIS_COMMIT,
         json: true,
         headers: {
             'Authorization': 'token ' + process.env.TOKEN,
             'User-Agent': 'Cesium'
         },
         body: {
             state: state,
             target_url: targetUrl,
             description: description,
             context: context
         }
     });
}

gulp.task('coverage', function(done) {
    var argv = yargs.argv;
    var webglStub = argv.webglStub ? argv.webglStub : false;
    var suppressPassed = argv.suppressPassed ? argv.suppressPassed : false;
    var failTaskOnError = argv.failTaskOnError ? argv.failTaskOnError : false;

    var folders = [];
    var browsers = ['Chrome'];
    if (argv.browsers) {
        browsers = argv.browsers.split(',');
    }

    var karma = new Karma.Server({
        configFile: karmaConfigFile,
        browsers: browsers,
        specReporter: {
            suppressErrorSummary: false,
            suppressFailed: false,
            suppressPassed: suppressPassed,
            suppressSkipped: true
        },
        preprocessors: {
            'Source/Core/**/*.js': ['karma-coverage-istanbul-instrumenter'],
            'Source/DataSources/**/*.js': ['karma-coverage-istanbul-instrumenter'],
            'Source/Renderer/**/*.js': ['karma-coverage-istanbul-instrumenter'],
            'Source/Scene/**/*.js': ['karma-coverage-istanbul-instrumenter'],
            'Source/Shaders/**/*.js': ['karma-coverage-istanbul-instrumenter'],
            'Source/Widgets/**/*.js': ['karma-coverage-istanbul-instrumenter'],
            'Source/Workers/**/*.js': ['karma-coverage-istanbul-instrumenter']
        },
        coverageIstanbulInstrumenter: {
            esModules: true
        },
        reporters: ['spec', 'coverage'],
        coverageReporter: {
            dir: 'Build/Coverage',
            subdir: function(browserName) {
                folders.push(browserName);
                return browserName;
            },
            includeAllSources: true
        },
        client: {
            captureConsole: verbose,
            args: [undefined, undefined, undefined, webglStub, undefined]
        }
    }, function(e) {
        var html = '<!doctype html><html><body><ul>';
        folders.forEach(function(folder) {
            html += '<li><a href="' + encodeURIComponent(folder) + '/index.html">' + folder + '</a></li>';
        });
        html += '</ul></body></html>';
        fs.writeFileSync('Build/Coverage/index.html', html);

        if (!process.env.TRAVIS) {
            folders.forEach(function(dir) {
                open('Build/Coverage/' + dir + '/index.html');
            });
        }
        return done(failTaskOnError ? e : undefined);
    });
    karma.start();
});

gulp.task('test', function(done) {
    var argv = yargs.argv;

    var enableAllBrowsers = argv.all ? true : false;
    var includeCategory = argv.include ? argv.include : '';
    var excludeCategory = argv.exclude ? argv.exclude : '';
    var webglValidation = argv.webglValidation ? argv.webglValidation : false;
    var webglStub = argv.webglStub ? argv.webglStub : false;
    var release = argv.release ? argv.release : false;
    var failTaskOnError = argv.failTaskOnError ? argv.failTaskOnError : false;
    var suppressPassed = argv.suppressPassed ? argv.suppressPassed : false;

    var browsers = ['Chrome'];
    if (argv.browsers) {
        browsers = argv.browsers.split(',');
    }

    var files = [
        { pattern: 'Specs/karma-main.js', included: true, type: 'module' },
        { pattern: 'Source/**', included: false, type: 'module' },
        { pattern: 'Specs/*.js', included: true, type: 'module' },
        { pattern: 'Specs/Core/**', included: true, type: 'module' },
        { pattern: 'Specs/Data/**', included: false },
        { pattern: 'Specs/DataSources/**', included: true, type: 'module' },
        { pattern: 'Specs/Renderer/**', included: true, type: 'module' },
        { pattern: 'Specs/Scene/**', included: true, type: 'module' },
        { pattern: 'Specs/ThirdParty/**', included: true, type: 'module' },
        { pattern: 'Specs/Widgets/**', included: true, type: 'module' },
        { pattern: 'Specs/TestWorkers/**', included: false }
    ];

    if (release) {
        files = [
            { pattern: 'Specs/Data/**', included: false },
            { pattern: 'Specs/ThirdParty/**', included: true, type: 'module' },
            { pattern: 'Specs/TestWorkers/**', included: false },
            { pattern: 'Build/Cesium/Cesium.js', included: true },
            { pattern: 'Build/Cesium/**', included: false },
            { pattern: 'Build/Specs/karma-main.js', included: true },
            { pattern: 'Build/Specs/Specs.js', included: true }
        ];
    }

    var karma = new Karma.Server({
        configFile: karmaConfigFile,
        browsers: browsers,
        specReporter: {
            suppressErrorSummary: false,
            suppressFailed: false,
            suppressPassed: suppressPassed,
            suppressSkipped: true
        },
        detectBrowsers: {
            enabled: enableAllBrowsers
        },
        logLevel: verbose ? Karma.constants.LOG_INFO : Karma.constants.LOG_ERROR,
        files: files,
        client: {
            captureConsole: verbose,
            args: [includeCategory, excludeCategory, webglValidation, webglStub, release]
        }
    }, function(e) {
        return done(failTaskOnError ? e : undefined);
    });
    karma.start();
});

gulp.task('convertToModules', function() {
    var requiresRegex = /([\s\S]*?(define|defineSuite|require)\((?:{[\s\S]*}, )?\[)([\S\s]*?)]([\s\S]*?function\s*)\(([\S\s]*?)\) {([\s\S]*)/;
    var noModulesRegex = /([\s\S]*?(define|defineSuite|require)\((?:{[\s\S]*}, )?\[?)([\S\s]*?)]?([\s\S]*?function\s*)\(([\S\s]*?)\) {([\s\S]*)/;
    var splitRegex = /,\s*/;

    var fsReadFile = Promise.promisify(fs.readFile);
    var fsWriteFile = Promise.promisify(fs.writeFile);

    var files = globby.sync(filesToConvertES6);

    return Promise.map(files, function(file) {
        return fsReadFile(file).then(function(contents) {
            contents = contents.toString();
            if (contents.startsWith('import')) {
                return;
            }

            var result = requiresRegex.exec(contents);

            if (result === null) {
                result = noModulesRegex.exec(contents);
                if (result === null) {
                    return;
                }
            }

            var names = result[3].split(splitRegex);
            if (names.length === 1 && names[0].trim() === '') {
                names.length = 0;
            }

            var i;
            for (i = 0; i < names.length; ++i) {
                if (names[i].indexOf('//') >= 0 || names[i].indexOf('/*') >= 0) {
                    console.log(file + ' contains comments in the require list.  Skipping so nothing gets broken.');
                    return;
                }
            }

            var identifiers = result[5].split(splitRegex);
            if (identifiers.length === 1 && identifiers[0].trim() === '') {
                identifiers.length = 0;
            }

            for (i = 0; i < identifiers.length; ++i) {
                if (identifiers[i].indexOf('//') >= 0 || identifiers[i].indexOf('/*') >= 0) {
                    console.log(file + ' contains comments in the require list.  Skipping so nothing gets broken.');
                    return;
                }
            }

            var requires = [];

            for (i = 0; i < names.length && i < identifiers.length; ++i) {
                requires.push({
                    name : names[i].trim(),
                    identifier : identifiers[i].trim()
                });
            }

            // Convert back to separate lists for the names and identifiers, and add
            // any additional names or identifiers that don't have a corresponding pair.
            var sortedNames = requires.map(function(item) {
                return item.name.slice(0, -1) + '.js\'';
            });
            for (i = sortedNames.length; i < names.length; ++i) {
                sortedNames.push(names[i].trim());
            }

            var sortedIdentifiers = requires.map(function(item) {
                return item.identifier;
            });
            for (i = sortedIdentifiers.length; i < identifiers.length; ++i) {
                sortedIdentifiers.push(identifiers[i].trim());
            }

            contents = '';
            if (sortedNames.length > 0) {
                for (var q = 0; q < sortedNames.length; q++) {
                    var modulePath = sortedNames[q];
                    if (file.startsWith('Specs')) {
                        modulePath = modulePath.substring(1, modulePath.length - 1);
                        var sourceDir = path.dirname(file);

                        if (modulePath.startsWith('Specs') || modulePath.startsWith('.')) {
                            var importPath = modulePath;
                            if (modulePath.startsWith('Specs')) {
                                importPath = path.relative(sourceDir, modulePath);
                                if (importPath[0] !== '.') {
                                    importPath = './' + importPath;
                                }
                            }
                            modulePath = '\'' + importPath + '\'';
                            contents += 'import ' + sortedIdentifiers[q] + ' from ' + modulePath + ';' + os.EOL;
                        } else {
                            modulePath = '\'' + path.relative(sourceDir, 'Source') + '/Cesium.js' + '\'';
                            if (sortedIdentifiers[q] === 'CesiumMath') {
                                contents += 'import { Math as CesiumMath } from ' + modulePath + ';' + os.EOL;
                            } else {
                                contents += 'import { ' + sortedIdentifiers[q] + ' } from ' + modulePath + ';' + os.EOL;
                            }
                        }
                    } else {
                        contents += 'import ' + sortedIdentifiers[q] + ' from ' + modulePath + ';' + os.EOL;
                    }
                }
            }

            var code;
            var codeAndReturn = result[6];
            if (file.endsWith('Spec.js')) {
                var indi = codeAndReturn.lastIndexOf('});');
                code = codeAndReturn.slice(0, indi);
                code = code.trim().replace("'use strict';" + os.EOL, '');
                contents += code + os.EOL;
            } else {
                var returnIndex = codeAndReturn.lastIndexOf('return');

                code = codeAndReturn.slice(0, returnIndex);
                code = code.trim().replace("'use strict';" + os.EOL, '');
                contents += code + os.EOL;

                var returnStatement = codeAndReturn.slice(returnIndex);
                contents += returnStatement.split(';')[0].replace('return ', 'export default ') + ';' + os.EOL;
            }

            return fsWriteFile(file, contents);
        });
    });
});

function combineCesium(debug, optimizer, combineOutput) {
    var plugins = [];

    if (!debug) {
        plugins.push(rollupPluginStripPragma({
            pragmas: ['debug']
        }));
    }
    if (optimizer === 'uglify2') {
        plugins.push(rollupPluginUglify.uglify());
    }

    return rollup.rollup({
        input: 'Source/Cesium.js',
        plugins: plugins,
        onwarn: rollupWarning
    }).then(function(bundle) {
        return bundle.write({
            format: 'umd',
            name: 'Cesium',
            file: path.join(combineOutput, 'Cesium.js')
        });
    });
}

function combineWorkers(debug, optimizer, combineOutput) {
    //This is done waterfall style for concurrency reasons.
    // Copy files that are already minified
    return globby(['Source/ThirdParty/Workers/draco*.js'])
        .then(function(files) {
            var stream = gulp.src(files, { base: 'Source' })
                .pipe(gulp.dest(combineOutput));
            return streamToPromise(stream);
        })
        .then(function () {
            return globby(['Source/Workers/cesiumWorkerBootstrapper.js',
                'Source/Workers/transferTypedArrayTest.js',
                'Source/ThirdParty/Workers/*.js',
                // Files are already minified, don't optimize
                '!Source/ThirdParty/Workers/draco*.js']);
        })
        .then(function(files) {
            return Promise.map(files, function(file) {
                return streamToPromise(gulp.src(file)
                    .pipe(gulpUglify())
                    .pipe(gulp.dest(path.dirname(path.join(combineOutput, path.relative('Source', file))))));
            }, {concurrency : concurrency});
        })
        .then(function() {
            return globby(['Source/WorkersES6/*.js']);
        })
        .then(function(files) {
            var plugins = [];

            if (!debug) {
                plugins.push(rollupPluginStripPragma({
                    pragmas: ['debug']
                }));
            }
            if (optimizer === 'uglify2') {
                plugins.push(rollupPluginUglify.uglify());
            }

            return rollup.rollup({
                input: files,
                plugins: plugins,
                onwarn: rollupWarning
            }).then(function(bundle) {
                return bundle.write({
                    dir: path.join(combineOutput, 'Workers'),
                    format: 'amd'
                });
            });
        });
}

function minifyCSS(outputDirectory) {
    streamToPromise(
        gulp.src('Source/**/*.css')
            .pipe(cleanCSS())
            .pipe(gulp.dest(outputDirectory))
    );
}

function minifyModules(outputDirectory) {
    return streamToPromise(gulp.src('Source/ThirdParty/google-earth-dbroot-parser.js')
        .pipe(gulpUglify())
        .pipe(gulp.dest(outputDirectory + '/ThirdParty/')));
}

function combineJavaScript(options) {
    var optimizer = options.optimizer;
    var outputDirectory = options.outputDirectory;
    var removePragmas = options.removePragmas;

    var combineOutput = path.join('Build', 'combineOutput', optimizer);
    var copyrightHeader = fs.readFileSync(path.join('Source', 'copyrightHeader.js'));

    var promise = Promise.join(
        combineCesium(!removePragmas, optimizer, combineOutput),
        combineWorkers(!removePragmas, optimizer, combineOutput),
        minifyModules(outputDirectory)
    );

    return promise.then(function() {
        var promises = [];

        //copy to build folder with copyright header added at the top
        var stream = gulp.src([combineOutput + '/**'])
            .pipe(gulpInsert.prepend(copyrightHeader))
            .pipe(gulp.dest(outputDirectory));

        promises.push(streamToPromise(stream));

        var everythingElse = ['Source/**', '!**/*.js', '!**/*.glsl'];
        if (optimizer === 'uglify2') {
            promises.push(minifyCSS(outputDirectory));
            everythingElse.push('!**/*.css');
        }

        stream = gulp.src(everythingElse, { nodir: true }).pipe(gulp.dest(outputDirectory));
        promises.push(streamToPromise(stream));

        return Promise.all(promises).then(function() {
            rimraf.sync(combineOutput);
        });
    });
}

function glslToJavaScript(minify, minifyStateFilePath) {
    fs.writeFileSync(minifyStateFilePath, minify);
    var minifyStateFileLastModified = fs.existsSync(minifyStateFilePath) ? fs.statSync(minifyStateFilePath).mtime.getTime() : 0;

// collect all currently existing JS files into a set, later we will remove the ones
// we still are using from the set, then delete any files remaining in the set.
    var leftOverJsFiles = {};

    globby.sync(['Source/Shaders/**/*.js', 'Source/ThirdParty/Shaders/*.js']).forEach(function(file) {
        leftOverJsFiles[path.normalize(file)] = true;
    });

    var builtinFunctions = [];
    var builtinConstants = [];
    var builtinStructs = [];

    var glslFiles = globby.sync(['Source/Shaders/**/*.glsl', 'Source/ThirdParty/Shaders/*.glsl']);
    glslFiles.forEach(function(glslFile) {
        glslFile = path.normalize(glslFile);
        var baseName = path.basename(glslFile, '.glsl');
        var jsFile = path.join(path.dirname(glslFile), baseName) + '.js';

        // identify built in functions, structs, and constants
        var baseDir = path.join('Source', 'Shaders', 'Builtin');
        if (glslFile.indexOf(path.normalize(path.join(baseDir, 'Functions'))) === 0) {
            builtinFunctions.push(baseName);
        }
        else if (glslFile.indexOf(path.normalize(path.join(baseDir, 'Constants'))) === 0) {
            builtinConstants.push(baseName);
        }
        else if (glslFile.indexOf(path.normalize(path.join(baseDir, 'Structs'))) === 0) {
            builtinStructs.push(baseName);
        }

        delete leftOverJsFiles[jsFile];

        var jsFileExists = fs.existsSync(jsFile);
        var jsFileModified = jsFileExists ? fs.statSync(jsFile).mtime.getTime() : 0;
        var glslFileModified = fs.statSync(glslFile).mtime.getTime();

        if (jsFileExists && jsFileModified > glslFileModified && jsFileModified > minifyStateFileLastModified) {
            return;
        }

        var contents = fs.readFileSync(glslFile, 'utf8');
        contents = contents.replace(/\r\n/gm, '\n');

        var copyrightComments = '';
        var extractedCopyrightComments = contents.match(/\/\*\*(?:[^*\/]|\*(?!\/)|\n)*?@license(?:.|\n)*?\*\//gm);
        if (extractedCopyrightComments) {
            copyrightComments = extractedCopyrightComments.join('\n') + '\n';
        }

        if (minify) {
            contents = glslStripComments(contents);
            contents = contents.replace(/\s+$/gm, '').replace(/^\s+/gm, '').replace(/\n+/gm, '\n');
            contents += '\n';
        }

        contents = contents.split('"').join('\\"').replace(/\n/gm, '\\n\\\n');
        contents = copyrightComments + '\
//This file is automatically rebuilt by the Cesium build process.\n\
export default "' + contents + '";\n';

        fs.writeFileSync(jsFile, contents);
    });

    // delete any left over JS files from old shaders
    Object.keys(leftOverJsFiles).forEach(function(filepath) {
        rimraf.sync(filepath);
    });

    var generateBuiltinContents = function(contents, builtins, path) {
        for (var i = 0; i < builtins.length; i++) {
            var builtin = builtins[i];
            contents.imports.push('import czm_' + builtin + ' from \'./' + path + '/' + builtin + '.js\'');
            contents.builtinLookup.push('czm_' + builtin + ' : ' + 'czm_' + builtin);
        }
    };

    //generate the JS file for Built-in GLSL Functions, Structs, and Constants
    var contents = {
        imports : [],
        builtinLookup: []
    };
    generateBuiltinContents(contents, builtinConstants, 'Constants');
    generateBuiltinContents(contents, builtinStructs, 'Structs');
    generateBuiltinContents(contents, builtinFunctions, 'Functions');

    var fileContents = '//This file is automatically rebuilt by the Cesium build process.\n' +
        contents.imports.join('\n') +
        '\n\nexport default {\n    ' + contents.builtinLookup.join(',\n    ') + '\n};\n';

    fs.writeFileSync(path.join('Source', 'Shaders', 'Builtin', 'CzmBuiltins.js'), fileContents);
}

function createCesiumJs() {
    var contents = `export var VERSION = '${version}';\n`;
    globby.sync(sourceFiles).forEach(function(file) {
        file = path.relative('Source', file);

        var moduleId = file;
        moduleId = filePathToModuleId(moduleId);

        var assignmentName = path.basename(file, path.extname(file));
        if (moduleId.indexOf('Shaders/') === 0) {
            assignmentName = '_shaders' + assignmentName;
        }
        assignmentName = assignmentName.replace(/(\.|-)/g, '_');
        contents += 'export { default as ' + assignmentName + " } from './" + moduleId + ".js';" + os.EOL;
    });

    fs.writeFileSync('Source/Cesium.js', contents);
}

function createSpecList() {
    var specFiles = globby.sync(['Specs/**/*Spec.js']);

    var contents = '';
    specFiles.forEach(function(file) {
        contents += "import './" + filePathToModuleId(file).replace('Specs/', '') + ".js';\n";
    });

    fs.writeFileSync(path.join('Specs', 'SpecList.js'), contents);
}

function createGalleryList() {
    var demoObjects = [];
    var demoJSONs = [];
    var output = path.join('Apps', 'Sandcastle', 'gallery', 'gallery-index.js');

    var fileList = ['Apps/Sandcastle/gallery/**/*.html'];
    if (noDevelopmentGallery) {
        fileList.push('!Apps/Sandcastle/gallery/development/**/*.html');
    }

    // On travis, the version is set to something like '1.43.0-branch-name-travisBuildNumber'
    // We need to extract just the Major.Minor version
    var majorMinor = packageJson.version.match(/^(.*)\.(.*)\./);
    var major = majorMinor[1];
    var minor = Number(majorMinor[2]) - 1; // We want the last release, not current release
    var tagVersion = major + '.' + minor;

    // Get an array of demos that were added since the last release.
    // This includes newly staged local demos as well.
    var newDemos = [];
    try {
        newDemos = child_process.execSync('git diff --name-only --diff-filter=A ' + tagVersion + ' Apps/Sandcastle/gallery/*.html', { stdio: ['pipe', 'pipe', 'ignore'] }).toString().trim().split('\n');
    } catch (e) {
        // On a Cesium fork, tags don't exist so we can't generate the list.
    }

    var helloWorld;
    globby.sync(fileList).forEach(function(file) {
        var demo = filePathToModuleId(path.relative('Apps/Sandcastle/gallery', file));

        var demoObject = {
            name : demo,
            isNew: newDemos.includes(file)
        };

        if (fs.existsSync(file.replace('.html', '') + '.jpg')) {
            demoObject.img = demo + '.jpg';
        }

        demoObjects.push(demoObject);

        if (demo === 'Hello World') {
            helloWorld = demoObject;
        }
    });

    demoObjects.sort(function(a, b) {
      if (a.name < b.name) {
        return -1;
      } else if (a.name > b.name) {
        return 1;
      }
      return 0;
    });

    var helloWorldIndex = Math.max(demoObjects.indexOf(helloWorld), 0);

    var i;
    for (i = 0; i < demoObjects.length; ++i) {
      demoJSONs[i] = JSON.stringify(demoObjects[i], null, 2);
    }

    var contents = '\
// This file is automatically rebuilt by the Cesium build process.\n\
var hello_world_index = ' + helloWorldIndex + ';\n\
var VERSION = \'' + version + '\';\n\
var gallery_demos = [' + demoJSONs.join(', ') + '];\n\
var has_new_gallery_demos = ' + (newDemos.length > 0 ? 'true;' : 'false;') + '\n';

    fs.writeFileSync(output, contents);

    // Compile CSS for Sandcastle
    return streamToPromise(gulp.src(path.join('Apps', 'Sandcastle', 'templates', 'bucketRaw.css'))
        .pipe(cleanCSS())
        .pipe(gulpRename('bucket.css'))
        .pipe(gulpInsert.prepend('/* This file is automatically rebuilt by the Cesium build process. */\n'))
        .pipe(gulp.dest(path.join('Apps', 'Sandcastle', 'templates'))));
}

function createJsHintOptions() {
    var primary = JSON.parse(fs.readFileSync(path.join('Apps', '.jshintrc'), 'utf8'));
    var gallery = JSON.parse(fs.readFileSync(path.join('Apps', 'Sandcastle', '.jshintrc'), 'utf8'));
    primary.jasmine = false;
    primary.predef = gallery.predef;
    primary.unused = gallery.unused;
    primary.esversion = gallery.esversion;

    var contents = '\
// This file is automatically rebuilt by the Cesium build process.\n\
var sandcastleJsHintOptions = ' + JSON.stringify(primary, null, 4) + ';\n';

    fs.writeFileSync(path.join('Apps', 'Sandcastle', 'jsHintOptions.js'), contents);
}

function buildSandcastle() {
    var appStream = gulp.src([
            'Apps/Sandcastle/**',
            '!Apps/Sandcastle/load-cesium-es6.js',
            '!Apps/Sandcastle/standalone.html',
            '!Apps/Sandcastle/images/**',
            '!Apps/Sandcastle/gallery/**.jpg'
        ])
        // Remove dev-only ES6 module loading for unbuilt Cesium
        .pipe(gulpReplace('    <script type="module" src="../load-cesium-es6.js"></script>', ''))
        .pipe(gulpReplace('nomodule', ''))
        // Fix relative paths for new location
        .pipe(gulpReplace('../../../Build', '../../..'))
        .pipe(gulpReplace('../../Source', '../../../Source'))
        .pipe(gulpReplace('../../ThirdParty', '../../../ThirdParty'))
        .pipe(gulpReplace('../../SampleData', '../../../../Apps/SampleData'))
        .pipe(gulpReplace('Build/Documentation', 'Documentation'))
        .pipe(gulp.dest('Build/Apps/Sandcastle'));

    var imageStream = gulp.src([
            'Apps/Sandcastle/gallery/**.jpg',
            'Apps/Sandcastle/images/**'
        ], {
            base: 'Apps/Sandcastle',
            buffer: false
        })
        .pipe(gulp.dest('Build/Apps/Sandcastle'));

    var standaloneStream = gulp.src([
        'Apps/Sandcastle/standalone.html'
        ])
        .pipe(gulpReplace('    <script type="module" src="load-cesium-es6.js"></script>', ''))
        .pipe(gulpReplace('nomodule', ''))
        .pipe(gulpReplace('../../Build', '../..'))
        .pipe(gulp.dest('Build/Apps/Sandcastle'));

    return streamToPromise(mergeStream(appStream, imageStream, standaloneStream));
}

function buildCesiumViewer() {
    var cesiumViewerOutputDirectory = 'Build/Apps/CesiumViewer';
    mkdirp.sync(cesiumViewerOutputDirectory);

    var promise = Promise.join(
        rollup.rollup({
            input: 'Apps/CesiumViewer/CesiumViewer.js',
            treeshake: {
                moduleSideEffects: false
            },
            plugins: [
                rollupPluginStripPragma({
                    pragmas: ['debug']
                }),
                rollupPluginUglify.uglify()
            ],
            onwarn: rollupWarning
        }).then(function(bundle) {
            return bundle.write({
                file: 'Build/Apps/CesiumViewer/CesiumViewer.js',
                format: 'iife'
            });
        })
    );

    promise = promise.then(function() {
        var copyrightHeader = fs.readFileSync(path.join('Source', 'copyrightHeader.js'));
        var stream = mergeStream(
            gulp.src('Build/Apps/CesiumViewer/CesiumViewer.js')
                .pipe(gulpInsert.prepend(copyrightHeader))
                .pipe(gulpReplace('../../Source', '.'))
                .pipe(gulp.dest(cesiumViewerOutputDirectory)),

            gulp.src('Apps/CesiumViewer/CesiumViewer.css')
                .pipe(cleanCSS())
                .pipe(gulpReplace('../../Source', '.'))
                .pipe(gulp.dest(cesiumViewerOutputDirectory)),

            gulp.src('Apps/CesiumViewer/index.html')
                .pipe(gulpReplace('type="module"', ''))
                .pipe(gulp.dest(cesiumViewerOutputDirectory)),

            gulp.src(['Apps/CesiumViewer/**',
                      '!Apps/CesiumViewer/index.html',
                      '!Apps/CesiumViewer/**/*.js',
                      '!Apps/CesiumViewer/**/*.css']),

            gulp.src(['Build/Cesium/Assets/**',
                      'Build/Cesium/Workers/**',
                      'Build/Cesium/ThirdParty/**',
                      'Build/Cesium/Widgets/**',
                      '!Build/Cesium/Widgets/**/*.css'],
                {
                    base : 'Build/Cesium',
                    nodir : true
                }),

            gulp.src(['Build/Cesium/Widgets/InfoBox/InfoBoxDescription.css'], {
                base : 'Build/Cesium'
            }),

            gulp.src(['web.config'])
        );

        return streamToPromise(stream.pipe(gulp.dest(cesiumViewerOutputDirectory)));
    });

    return promise;
}

function filePathToModuleId(moduleId) {
    return moduleId.substring(0, moduleId.lastIndexOf('.')).replace(/\\/g, '/');
}