Merge pull request #248 from hackmdio/file-upload-options

Support other options for image uploading

.gitignore

@@ -25,3 +25,6 @@ public/js/config.js
 # ignore webpack build
 public/build
 public/views/build
+
+public/uploads/*
+!public/uploads/.gitkeep

README.md

@@ -131,6 +131,11 @@ Environment variables (will overwrite other server configs)
 | HMD_GOOGLE_CLIENTID | no example | Google API client id |
 | HMD_GOOGLE_CLIENTSECRET | no example | Google API client secret |
 | HMD_IMGUR_CLIENTID | no example | Imgur API client id |
+| HMD_IMAGE_UPLOAD_TYPE | `imgur`, `s3` or `filesystem` | Where to upload image. For S3, see our [S3 Image Upload Guide](docs/guides/s3-image-upload.md) |
+| HMD_S3_ACCESS_KEY_ID | no example | AWS access key id |
+| HMD_S3_SECRET_ACCESS_KEY | no example | AWS secret key |
+| HMD_S3_REGION | `ap-northeast-1` | AWS S3 region |
+| HMD_S3_BUCKET | no example | AWS S3 bucket name |
 
 Server settings `config.json`
 ---
@@ -166,6 +171,8 @@ Server settings `config.json`
 | heartbeatinterval | `5000` | socket.io heartbeat interval |
 | heartbeattimeout | `10000` | socket.io heartbeat timeout |
 | documentmaxlength | `100000` | note max length |
+| imageUploadType | `imgur`(default), `s3` or `filesystem` | Where to upload image
+| s3 | `{ "accessKeyId": "YOUR_S3_ACCESS_KEY_ID", "secretAccessKey": "YOUR_S3_ACCESS_KEY", "region": "YOUR_S3_REGION", "bucket": "YOUR_S3_BUCKET_NAME" }` | When `imageUploadType` be setted to `s3`, you would also need to setup this key, check our [S3 Image Upload Guide](docs/guides/s3-image-upload.md) |
 
 Third-party integration api key settings
 ---

app.js

@@ -406,33 +406,102 @@ app.get('/me', function (req, res) {
         });
     }
 });
-//upload to imgur
+
+//upload image
 app.post('/uploadimage', function (req, res) {
     var form = new formidable.IncomingForm();
+
+    form.keepExtensions = true;
+
+    if (config.imageUploadType === 'filesystem') {
+        form.uploadDir = "public/uploads";
+    }
+
+    function preprocessImage(path) {
+        return new Promise((resolve) => {
+            var oldFile = `${path}-old`;
+            fs.rename(path, oldFile, function() {
+                var sharp = require('sharp');
+                sharp(oldFile).toFile(path).then(() => {
+                    fs.unlink(oldFile, function() {
+                        resolve(path);
+                    })
+                });
+            });
+        });
+    }
+
     form.parse(req, function (err, fields, files) {
         if (err || !files.image || !files.image.path) {
             response.errorForbidden(res);
         } else {
-            if (config.debug)
-                logger.info('SERVER received uploadimage: ' + JSON.stringify(files.image));
-            imgur.setClientId(config.imgur.clientID);
-            try {
-                imgur.uploadFile(files.image.path)
-                    .then(function (json) {
-                        if (config.debug)
-                            logger.info('SERVER uploadimage success: ' + JSON.stringify(json));
+            preprocessImage(files.image.path).then(() => {
+                if (config.debug)
+                    logger.info('SERVER received uploadimage: ' + JSON.stringify(files.image));
+
+                var path = require('path');
+                try {
+                    switch (config.imageUploadType) {
+                    case 'filesystem':
                         res.send({
-                            link: json.data.link.replace(/^http:\/\//i, 'https://')
+                            link: path.join(config.serverurl, files.image.path.match(/^public(.+$)/)[1])
                         });
-                    })
-                    .catch(function (err) {
-                        logger.error(err);
-                        return res.status(500).end('upload image error');
-                    });
-            } catch (err) {
+
+                        break;
+
+                    case 's3':
+                        var AWS = require('aws-sdk');
+                        var awsConfig = new AWS.Config(config.s3);
+                        var s3 = new AWS.S3(awsConfig);
+
+                        fs.readFile(files.image.path, function (err, buffer) {
+                            var params = {
+                                Bucket: 'hackmd',
+                                Key: path.join('uploads', path.basename(files.image.path)),
+                                Body: buffer
+                            };
+
+                            s3.putObject(params, function (err, data) {
+                                if (err) {
+                                    logger.error(err);
+                                    res.status(500).end('upload image error');
+                                } else {
+                                    res.send({
+                                        link: `https://s3-${config.s3.region}.amazonaws.com/${config.s3bucket}/${params.Key}`
+                                    });
+                                }
+                            });
+
+                        });
+
+                        break;
+
+                    case 'imgur':
+                    default:
+                        imgur.setClientId(config.imgur.clientID);
+                        imgur.uploadFile(files.image.path)
+                            .then(function (json) {
+                                if (config.debug)
+                                    logger.info('SERVER uploadimage success: ' + JSON.stringify(json));
+                                res.send({
+                                    link: json.data.link.replace(/^http:\/\//i, 'https://')
+                                });
+                            })
+                            .catch(function (err) {
+                                logger.error(err);
+                                return res.status(500).end('upload image error');
+                            });
+                        break;
+                    }
+                } catch (err) {
+                    logger.error(err);
+                    return res.status(500).end('upload image error');
+                }
+
+            }).catch((err) => {
                 logger.error(err);
-                return res.status(500).end('upload image error');
-            }
+                return res.status(500).end('process image error');
+            });
         }
     });
 });

app.json

@@ -107,5 +107,13 @@
     },
     "addons": [
         "heroku-postgresql"
+    ],
+    "buildpacks": [
+        {
+            "url": "https://github.com/alex88/heroku-buildpack-vips"
+        },
+        {
+            "url": "https://github.com/heroku/heroku-buildpack-nodejs"
+        }
     ]
 }

docs/guides/s3-image-upload.md

@@ -0,0 +1,81 @@
+# Guide - Setup HackMD S3 image upload
+
+1. Go to [AWS S3 console](https://console.aws.amazon.com/s3/home) and create a new bucket.
+
+    ![create-bucket](images/s3-image-upload/create-bucket.png)
+
+2. Click on bucket, select **Properties**  on the side panel, and find **Permission** section. Click **Edit bucket policy**.
+
+    ![bucket-property](images/s3-image-upload/bucket-property.png)
+
+3. Enter the following policy, replace `bucket_name` with your bucket name:
+
+    ![bucket-policy-editor](images/s3-image-upload/bucket-policy-editor.png)
+
+    ```json
+    {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Principal": "*",
+                "Action": "s3:GetObject",
+                "Resource": "arn:aws:s3:::bucket_name/uploads/*"
+            }
+        ]
+    }
+    ```
+
+4. Go to IAM console and create a new IAM user. Remember your user credentials(`key`/`access token`)
+
+5. Enter user page, select **Permission** tab, look at **Inline Policies** section, and click **Create User Policy**
+
+    ![iam-user](images/s3-image-upload/iam-user.png)
+
+6. Select **Custom Policy**
+
+    ![custom-policy](images/s3-image-upload/custom-policy.png)
+
+7. Enter the following policy, replace `bucket_name` with your bucket name:
+
+    ![review-policy](images/s3-image-upload/review-policy.png)
+
+    ```json
+    {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": [
+                    "s3:*"
+                ],
+                "Resource": [
+                    "arn:aws:s3:::bucket_name/uploads/*"
+                ]
+            }
+        ]
+    }
+    ```
+
+8. Edit `config.json` and set following keys:
+
+    ```javascript
+    {
+        "production": {
+            ...
+            "imageUploadType": "s3",
+            "s3": {
+                "accessKeyId": "YOUR_S3_ACCESS_KEY_ID",
+                "secretAccessKey": "YOUR_S3_ACCESS_KEY",
+                "region": "YOUR_S3_REGION", // example: ap-northeast-1
+                "bucket": "YOUR_S3_BUCKET_NAME"
+            }
+        }
+    }
+    ```
+
+9. In additional to edit `config.json` directly, you could also try [environment variable](https://github.com/hackmdio/hackmd#environment-variables-will-overwrite-other-server-configs).
+
+## Related Tools
+
+* [AWS Policy Generator](http://awspolicygen.s3.amazonaws.com/policygen.html)

lib/config.js

@@ -56,6 +56,17 @@ var heartbeattimeout = config.heartbeattimeout || 10000;
 // document
 var documentmaxlength = config.documentmaxlength || 100000;
 
+// image upload setting, available options are imgur/s3/filesystem
+var imageUploadType = process.env.HMD_IMAGE_UPLOAD_TYPE || config.imageUploadType || 'imgur';
+
+config.s3 = config.s3 || {};
+var s3 = {
+    accessKeyId: process.env.HMD_S3_ACCESS_KEY_ID || config.s3.accessKeyId,
+    secretAccessKey: process.env.HMD_S3_SECRET_ACCESS_KEY || config.s3.secretAccessKey,
+    region: process.env.HMD_S3_REGION || config.s3.region
+}
+var s3bucket = process.env.HMD_S3_BUCKET || config.s3.bucket;
+
 // auth
 var facebook = (process.env.HMD_FACEBOOK_CLIENTID && process.env.HMD_FACEBOOK_CLIENTSECRET) ? {
     clientID: process.env.HMD_FACEBOOK_CLIENTID,
@@ -139,5 +150,8 @@ module.exports = {
     gitlab: gitlab,
     dropbox: dropbox,
     google: google,
-    imgur: imgur
+    imgur: imgur,
+    imageUploadType: imageUploadType,
+    s3: s3,
+    s3bucket: s3bucket
 };

package.json

@@ -5,7 +5,7 @@
   "main": "app.js",
   "license": "MIT",
   "scripts": {
-    "dev": "webpack --config webpack.config.js --progress --colors --watch",
+    "dev": "webpack --config webpack.config.js --progress --colors --watch & nodemon app.js",
     "build": "webpack --config webpack.production.js --progress --colors",
     "assets:install": "bower install",
     "postinstall": "bin/heroku",
@@ -14,6 +14,7 @@
   "dependencies": {
     "Idle.Js": "github:shawnmclean/Idle.js",
     "async": "^2.0.1",
+    "aws-sdk": "^2.7.0",
     "blueimp-md5": "^2.4.0",
     "body-parser": "^1.15.2",
     "bootstrap": "^3.3.7",
@@ -95,6 +96,7 @@
     "sequelize": "^3.24.3",
     "select2": "^3.5.2-browserify",
     "sequelize-cli": "^2.4.0",
+    "sharp": "^0.16.2",
     "shortid": "2.2.6",
     "socket.io": "~1.6.0",
     "socket.io-client": "~1.6.0",
@@ -149,6 +151,7 @@
     "json-loader": "^0.5.4",
     "less": "^2.7.1",
     "less-loader": "^2.2.3",
+    "nodemon": "^1.11.0",
     "optimize-css-assets-webpack-plugin": "^1.3.0",
     "script-loader": "^0.7.0",
     "style-loader": "^0.13.1",