Merge pull request #61 from wjordan/external_sources

Support simple external sources through environment variables

README.md

@@ -131,6 +131,9 @@ The available variables are as follows:
 
   // Display an image if a 404 request is encountered from a source
   IMAGE_404: null
+
+  // Whitelist arbitrary HTTP source prefixes using EXTERNAL_SOURCE_*
+  EXTERNAL_SOURCE_WIKIPEDIA: 'https://upload.wikimedia.org/wikipedia/'
 ```
 
 
@@ -219,6 +222,20 @@ translates to:
 
 It is possible to bring images in from external sources and store them behind your own CDN. This is very useful when it comes to things like Facebook or Vimeo which have very inconsistent load times. Each external source can still enable any of the modification parameters list above.
 
+In addition to the provided external sources, you can easily add your own basic external sources using `EXTERNAL_SOURCE_*` environment variables. For example, to add Wikipedia as an external source, set the following environment variable:
+
+```
+EXTERNAL_SOURCE_WIKIPEDIA: 'https://upload.wikimedia.org/wikipedia/'
+```
+
+Then you can request images beginning with the provided path using the `ewikipedia` modifier, eg:
+
+    http://my.cdn.com/ewikipedia/en/7/70/Example.png
+
+translates to:
+
+    https://upload.wikimedia.org/wikipedia/en/7/70/Example.png
+
 It is worth noting that Twitter requires a full set of credentials as you need to poll their API in order to return profile pics.
 
 A shorter expiry on images from social sources can also be set via `IMAGE_EXPIRY_SHORT` env var so they expiry at a faster rate than other images.

src/config/environment_vars.js

@@ -59,7 +59,10 @@ vars = {
   LOCAL_FILE_PATH: process.cwd(),
 
   // Display an image if a 404 request is encountered from a source
-  IMAGE_404: null
+  IMAGE_404: null,
+
+  // Whitelist arbitrary HTTP source prefixes using EXTERNAL_SOURCE_*
+  EXTERNAL_SOURCE_WIKIPEDIA: 'https://upload.wikimedia.org/wikipedia/'
 
 };
 
@@ -84,6 +87,13 @@ _.forEach(vars, function(value, key){
 
 });
 
+// Add external sources from environment vars
+vars.externalSources = {};
+Object.keys(vars).concat(Object.keys(process.env)).filter(function(key) {
+  return (/^EXTERNAL_SOURCE_/).test(key);
+}).forEach(function(key) {
+  vars.externalSources[key.substr('EXTERNAL_SOURCE_'.length).toLowerCase()] = process.env[key] || vars[key];
+});
 
 // A few helpers to quickly determine the environment
 vars.development = vars.NODE_ENV === 'development';

src/image.js

@@ -142,6 +142,9 @@ Image.prototype.getFile = function(){
   if (_.has(this.modifiers, 'external')){
     if (_.has(sources, this.modifiers.external)){
       streamType = this.modifiers.external;
+    } else if (_.has(env.externalSources, this.modifiers.external)) {
+      Stream = sources.external;
+      return new Stream(this, this.modifiers.external, env.externalSources[this.modifiers.external]);
     }
   }
 

src/lib/modifiers.js

@@ -58,8 +58,8 @@ string     = require('../utils/string');
 filters    = require('../streams/filters');
 sources    = require('../streams/sources');
 filterKeys = _.keys(filters);
-sourceKeys = _.keys(sources);
 environment = require('../config/environment_vars');
+sourceKeys = _.keys(sources).concat(_.keys(environment.externalSources));
 fs         = require('fs');
 
 

src/streams/sources/external.js

@@ -0,0 +1,76 @@
+// Fetches an image from an external URL
+
+'use strict';
+
+var stream, util, request;
+
+stream  = require('stream');
+util    = require('util');
+request = require('request');
+
+function contentLength(bufs){
+  return bufs.reduce(function(sum, buf){
+    return sum + buf.length;
+  }, 0);
+}
+
+function External(image, key, prefix){
+  /* jshint validthis:true */
+  if (!(this instanceof External)){
+    return new External(image, key, prefix);
+  }
+  stream.Readable.call(this, { objectMode : true });
+  this.image = image;
+  this.ended = false;
+  this.key = key;
+  this.prefix = prefix;
+}
+
+util.inherits(External, stream.Readable);
+
+External.prototype._read = function(){
+  var _this = this,
+    url,
+    fbStream,
+    bufs = [];
+
+  if ( this.ended ){ return; }
+
+  // pass through if there is an error on the image object
+  if (this.image.isError()){
+    this.ended = true;
+    this.push(this.image);
+    return this.push(null);
+  }
+
+  url = this.prefix + '/' + this.image.path;
+
+  this.image.log.time(this.key);
+
+  fbStream = request.get(url);
+  fbStream.on('data', function(d){ bufs.push(d); });
+  fbStream.on('error', function(err){
+    _this.image.error = new Error(err);
+  });
+  fbStream.on('response', function(response) {
+    if (response.statusCode !== 200) {
+      _this.image.error = new Error('Error ' + response.statusCode + ':');
+    }
+  });
+  fbStream.on('end', function(){
+    _this.image.log.timeEnd(_this.key);
+    if(_this.image.isError()) {
+      _this.image.error.message += Buffer.concat(bufs);
+    } else {
+      _this.image.contents = Buffer.concat(bufs);
+    }
+    _this.image.originalContentLength = contentLength(bufs);
+    _this.ended = true;
+    _this.push(_this.image);
+    _this.push(null);
+  });
+
+};
+
+
+module.exports = External;

test/index.html

@@ -123,6 +123,10 @@ <h2>External Sources</h2>
     <p>h200-eyoutube</p>
   </div>
 
+  <div class="sample-img">
+    <img src="/h200-ewikipedia/en/7/70/Example.png">
+    <p>h200-ewikipedia</p>
+  </div>
 
   <h2>Filters</h2>