From a7a104604c3d45bbee900b15d06cbf74c336b8a3 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 3 Apr 2019 14:17:54 +0200 Subject: [PATCH 1/3] fix(gateway): streaming compressed payload This change simplifies code responsible for streaming response and makes the streaming actually work by telling the payload compression stream to flush its content on every read(). (previous version was buffering entire thing in Hapi's compressor memory) We also do content-type detection based on the beginning of the stream by peeking at first `fileType.minimumBytes` bytes. License: MIT Signed-off-by: Marcin Rataj --- package.json | 1 + src/http/gateway/resources/gateway.js | 99 ++++++++++++++------------- test/gateway/index.js | 6 ++ 3 files changed, 57 insertions(+), 49 deletions(-) diff --git a/package.json b/package.json index b892e1662a..e0bc362682 100644 --- a/package.json +++ b/package.json @@ -90,6 +90,7 @@ "bl": "^3.0.0", "boom": "^7.2.0", "bs58": "^4.0.1", + "buffer-peek-stream": "^1.0.1", "byteman": "^1.3.5", "cid-tool": "~0.2.0", "cids": "~0.5.8", diff --git a/src/http/gateway/resources/gateway.js b/src/http/gateway/resources/gateway.js index cd2ce1c433..6f1d324675 100644 --- a/src/http/gateway/resources/gateway.js +++ b/src/http/gateway/resources/gateway.js @@ -3,13 +3,12 @@ const debug = require('debug') const log = debug('ipfs:http-gateway') log.error = debug('ipfs:http-gateway:error') -const pull = require('pull-stream') -const pushable = require('pull-pushable') -const toStream = require('pull-stream-to-stream') + const fileType = require('file-type') const mime = require('mime-types') const { PassThrough } = require('readable-stream') const Boom = require('boom') +const peek = require('buffer-peek-stream') const { resolver } = require('ipfs-http-response') const PathUtils = require('../utils/path') @@ -30,6 +29,20 @@ function detectContentType (ref, chunk) { return mime.contentType(mimeType) } +// Enable streaming of compressed payload +// https://github.com/hapijs/hapi/issues/3599 +class ResponseStream extends PassThrough { + _read (size) { + super._read(size) + if (this._compressor) { + this._compressor.flush() + } + } + setCompressor (compressor) { + this._compressor = compressor + } +} + module.exports = { checkCID (request, h) { if (!request.params.cid) { @@ -85,58 +98,46 @@ module.exports = { return h.redirect(PathUtils.removeTrailingSlash(ref)).permanent(true) } - return new Promise((resolve, reject) => { - let pusher - let started = false - - pull( - ipfs.catPullStream(data.cid), - pull.drain( - chunk => { - if (!started) { - started = true - pusher = pushable() - const res = h.response(toStream.source(pusher).pipe(new PassThrough())) - - // Etag maps directly to an identifier for a specific version of a resource - res.header('Etag', `"${data.cid}"`) + const rawStream = ipfs.catReadableStream(data.cid) + const responseStream = new ResponseStream() + + // Pass-through Content-Type sniffing over initial bytes + const contentType = await new Promise((resolve, reject) => { + try { + const peekBytes = fileType.minimumBytes + peek(rawStream, peekBytes, (err, streamHead, outputStream) => { + if (err) { + log.error(err) + return reject(err) + } + outputStream.pipe(responseStream) + resolve(detectContentType(ref, streamHead)) + }) + } catch (err) { + log.error(err) + reject(err) + } + }) - // Set headers specific to the immutable namespace - if (ref.startsWith('/ipfs/')) { - res.header('Cache-Control', 'public, max-age=29030400, immutable') - } + const res = h.response(responseStream) - const contentType = detectContentType(ref, chunk) + // Etag maps directly to an identifier for a specific version of a resource + res.header('Etag', `"${data.cid}"`) - log('ref ', ref) - log('mime-type ', contentType) + // Set headers specific to the immutable namespace + if (ref.startsWith('/ipfs/')) { + res.header('Cache-Control', 'public, max-age=29030400, immutable') + } - if (contentType) { - log('writing content-type header') - res.header('Content-Type', contentType) - } + log('ref ', ref) + log('content-type ', contentType) - resolve(res) - } - pusher.push(chunk) - }, - err => { - if (err) { - log.error(err) - - // We already started flowing, abort the stream - if (started) { - return pusher.end(err) - } - - return reject(err) - } + if (contentType) { + log('writing content-type header') + res.header('Content-Type', contentType) + } - pusher.end() - } - ) - ) - }) + return res }, afterHandler (request, h) { diff --git a/test/gateway/index.js b/test/gateway/index.js index f8836700fb..51c59ebc70 100644 --- a/test/gateway/index.js +++ b/test/gateway/index.js @@ -156,10 +156,15 @@ describe('HTTP Gateway', function () { expect(res.statusCode).to.equal(200) expect(res.rawPayload).to.eql(bigFile) + expect(res.headers['x-ipfs-path']).to.equal(`/ipfs/${bigFileHash}`) + expect(res.headers['etag']).to.equal(`"${bigFileHash}"`) + expect(res.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + expect(res.headers['content-type']).to.equal('application/octet-stream') }) it('load a jpg file', async () => { const kitty = 'QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ/cat.jpg' + const kittyDirectCid = 'Qmd286K6pohQcTKYqnS1YhWrCiS4gz7Xi34sdwMe9USZ7u' const res = await gateway.inject({ method: 'GET', @@ -169,6 +174,7 @@ describe('HTTP Gateway', function () { expect(res.statusCode).to.equal(200) expect(res.headers['content-type']).to.equal('image/jpeg') expect(res.headers['x-ipfs-path']).to.equal('/ipfs/' + kitty) + expect(res.headers['etag']).to.equal(`"${kittyDirectCid}"`) expect(res.headers['cache-control']).to.equal('public, max-age=29030400, immutable') expect(res.headers.etag).to.equal('"Qmd286K6pohQcTKYqnS1YhWrCiS4gz7Xi34sdwMe9USZ7u"') expect(res.headers.suborigin).to.equal('ipfs000bafybeidsg6t7ici2osxjkukisd5inixiunqdpq2q5jy4a2ruzdf6ewsqk4') From 373eedcb17ac6d2580a117709a8998ec39b59e58 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 26 Apr 2019 12:48:15 +0200 Subject: [PATCH 2/3] feat(gateway): range and conditional requests - Switched from deprecated `hapi` and `joi` to `@hapi/hapi` and `@hapi/joi` - Added support for Conditional Requests (RFC7232) - Returning `304 Not Modified` if `If-None-Match` is a CID matching `Etag` - Added `Last-Modified` to `/ipfs/` responses (improves client-side caching) - Always returning `304 Not Modified` when `If-Modified-Since` is present for immutable `/ipfs/` - Added support for Byte Range requests (RFC7233, Section-2.1) - Added support for `?filename=` parameter (improves downloads of raw cids) License: MIT Signed-off-by: Marcin Rataj --- package.json | 1 + src/http/gateway/resources/gateway.js | 73 +++++++- src/http/gateway/routes/gateway.js | 3 + test/gateway/index.js | 239 +++++++++++++++++++++++++- 4 files changed, 311 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index e0bc362682..f818b7163b 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "stream-to-promise": "^2.2.0" }, "dependencies": { + "@hapi/ammo": "^3.1.0", "@hapi/hapi": "^18.3.1", "@hapi/joi": "^15.0.1", "async": "^2.6.1", diff --git a/src/http/gateway/resources/gateway.js b/src/http/gateway/resources/gateway.js index 6f1d324675..9b903625ac 100644 --- a/src/http/gateway/resources/gateway.js +++ b/src/http/gateway/resources/gateway.js @@ -8,6 +8,7 @@ const fileType = require('file-type') const mime = require('mime-types') const { PassThrough } = require('readable-stream') const Boom = require('boom') +const Ammo = require('@hapi/ammo') // HTTP Range processing utilities const peek = require('buffer-peek-stream') const { resolver } = require('ipfs-http-response') @@ -98,7 +99,47 @@ module.exports = { return h.redirect(PathUtils.removeTrailingSlash(ref)).permanent(true) } - const rawStream = ipfs.catReadableStream(data.cid) + // Support If-None-Match & Etag (Conditional Requests from RFC7232) + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag + const etag = `"${data.cid}"` + const cachedEtag = request.headers['if-none-match'] + if (cachedEtag === etag || cachedEtag === `W/${etag}`) { + return h.response().code(304) // Not Modified + } + + // Immutable content produces 304 Not Modified for all values of If-Modified-Since + if (ref.startsWith('/ipfs/') && request.headers['if-modified-since']) { + return h.response().code(304) // Not Modified + } + + // This necessary to set correct Content-Length and validate Range requests + // Note: we need `size` (raw data), not `cumulativeSize` (data + DAGNodes) + const { size } = await ipfs.files.stat(`/ipfs/${data.cid}`) + + // Handle Byte Range requests (https://tools.ietf.org/html/rfc7233#section-2.1) + const catOptions = {} + let rangeResponse = false + if (request.headers.range) { + // If-Range is respected (when present), but we compare it only against Etag + // (Last-Modified date is too weak for IPFS use cases) + if (!request.headers['if-range'] || request.headers['if-range'] === etag) { + const ranges = Ammo.header(request.headers.range, size) + if (!ranges) { + const error = Boom.rangeNotSatisfiable() + error.output.headers['content-range'] = `bytes */${size}` + throw error + } + + if (ranges.length === 1) { // Ignore requests for multiple ranges (hard to map to ipfs.cat and not used in practice) + rangeResponse = true + const range = ranges[0] + catOptions.offset = range.from + catOptions.length = (range.to - range.from + 1) + } + } + } + + const rawStream = ipfs.catReadableStream(data.cid, catOptions) const responseStream = new ResponseStream() // Pass-through Content-Type sniffing over initial bytes @@ -119,10 +160,11 @@ module.exports = { } }) - const res = h.response(responseStream) + const res = h.response(responseStream).code(rangeResponse ? 206 : 200) // Etag maps directly to an identifier for a specific version of a resource - res.header('Etag', `"${data.cid}"`) + // and enables smart client-side caching thanks to If-None-Match + res.header('etag', etag) // Set headers specific to the immutable namespace if (ref.startsWith('/ipfs/')) { @@ -137,15 +179,38 @@ module.exports = { res.header('Content-Type', contentType) } + if (rangeResponse) { + const from = catOptions.offset + const to = catOptions.offset + catOptions.length - 1 + res.header('Content-Range', `bytes ${from}-${to}/${size}`) + res.header('Content-Length', catOptions.length) + } else { + // Announce support for Range requests + res.header('Accept-Ranges', 'bytes') + res.header('Content-Length', size) + } + + // Support Content-Disposition via ?filename=foo parameter + // (useful for browser vendor to download raw CID into custom filename) + // Source: https://github.com/ipfs/go-ipfs/blob/v0.4.20/core/corehttp/gateway_handler.go#L232-L236 + if (request.query.filename) { + res.header('Content-Disposition', `inline; filename*=UTF-8''${encodeURIComponent(request.query.filename)}`) + } + return res }, afterHandler (request, h) { const { response } = request - if (response.statusCode === 200) { + // Add headers to successfult responses (regular or range) + if (response.statusCode === 200 || response.statusCode === 206) { const { ref } = request.pre.args response.header('X-Ipfs-Path', ref) if (ref.startsWith('/ipfs/')) { + // "set modtime to a really long time ago, since files are immutable and should stay cached" + // Source: https://github.com/ipfs/go-ipfs/blob/v0.4.20/core/corehttp/gateway_handler.go#L228-L229 + response.header('Last-Modified', 'Thu, 01 Jan 1970 00:00:01 GMT') + // Suborigins: https://github.com/ipfs/in-web-browsers/issues/66 const rootCid = ref.split('/')[2] const ipfsOrigin = cidToString(rootCid, { base: 'base32' }) response.header('Suborigin', 'ipfs000' + ipfsOrigin) diff --git a/src/http/gateway/routes/gateway.js b/src/http/gateway/routes/gateway.js index eb8543d465..4fe5d640b0 100644 --- a/src/http/gateway/routes/gateway.js +++ b/src/http/gateway/routes/gateway.js @@ -10,6 +10,9 @@ module.exports = { pre: [ { method: resources.gateway.checkCID, assign: 'args' } ], + response: { + ranges: false // disable built-in support, we do it manually + }, ext: { onPostHandler: { method: resources.gateway.afterHandler } } diff --git a/test/gateway/index.js b/test/gateway/index.js index 51c59ebc70..3c13a7452d 100644 --- a/test/gateway/index.js +++ b/test/gateway/index.js @@ -1,4 +1,5 @@ /* eslint-env mocha */ +/* eslint dot-notation: 0, dot-notation: 0, quote-props: 0 */ 'use strict' const chai = require('chai') @@ -122,6 +123,8 @@ describe('HTTP Gateway', function () { expect(res.rawPayload).to.eql(Buffer.from('hello world' + '\n')) expect(res.payload).to.equal('hello world' + '\n') expect(res.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + expect(res.headers['content-length']).to.equal(res.rawPayload.length).to.equal(12) + expect(res.headers['last-modified']).to.equal('Thu, 01 Jan 1970 00:00:01 GMT') expect(res.headers.etag).to.equal('"QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o"') expect(res.headers['x-ipfs-path']).to.equal('/ipfs/QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o') expect(res.headers.suborigin).to.equal('ipfs000bafybeicg2rebjoofv4kbyovkw7af3rpiitvnl6i7ckcywaq6xjcxnc2mby') @@ -146,7 +149,72 @@ describe('HTTP Gateway', function () { }) */ - it('stream a large file', async () => { + it('return 304 Not Modified if client announces cached CID in If-None-Match', async () => { + const cid = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + + // Get file first to simulate caching it and reading Etag + const resFirst = await gateway.inject({ + method: 'GET', + url: '/ipfs/' + cid + }) + expect(resFirst.statusCode).to.equal(200) + expect(resFirst.headers['etag']).to.equal(`"${cid}"`) + expect(resFirst.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + + // second request, this time announcing we have bigFileHash already in cache + const resSecond = await gateway.inject({ + method: 'GET', + url: '/ipfs/' + cid, + headers: { + 'If-None-Match': resFirst.headers.etag + } + }) + + // expect HTTP 304 Not Modified without payload + expect(resSecond.statusCode).to.equal(304) + expect(resSecond.rawPayload).to.be.empty() + }) + + it('return 304 Not Modified if /ipfs/ was requested with any If-Modified-Since', async () => { + const cid = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + + // Get file first to simulate caching it and reading Etag + const resFirst = await gateway.inject({ + method: 'GET', + url: '/ipfs/' + cid + }) + expect(resFirst.statusCode).to.equal(200) + expect(resFirst.headers['etag']).to.equal(`"${cid}"`) + expect(resFirst.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + + // second request, this time with If-Modified-Since equal present + const resSecond = await gateway.inject({ + method: 'GET', + url: '/ipfs/' + cid, + headers: { + 'If-Modified-Since': new Date().toUTCString() + } + }) + + // expect HTTP 304 Not Modified without payload + expect(resSecond.statusCode).to.equal(304) + expect(resSecond.rawPayload).to.be.empty() + }) + + it('return proper Content-Disposition if ?filename=foo is included in URL', async () => { + const cid = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + + // Get file first to simulate caching it and reading Etag + const resFirst = await gateway.inject({ + method: 'GET', + url: `/ipfs/${cid}?filename=pretty-name-in-utf8-%C3%B3%C3%B0%C5%9B%C3%B3%C3%B0%C5%82%C4%85%C5%9B%C5%81.txt` + }) + expect(resFirst.statusCode).to.equal(200) + expect(resFirst.headers['etag']).to.equal(`"${cid}"`) + expect(resFirst.headers['content-disposition']).to.equal(`inline; filename*=UTF-8''pretty-name-in-utf8-%C3%B3%C3%B0%C5%9B%C3%B3%C3%B0%C5%82%C4%85%C5%9B%C5%81.txt`) + }) + + it('load a big file (15MB)', async () => { const bigFileHash = 'Qme79tX2bViL26vNjPsF3DP1R9rMKMvnPYJiKTTKPrXJjq' const res = await gateway.inject({ @@ -156,12 +224,172 @@ describe('HTTP Gateway', function () { expect(res.statusCode).to.equal(200) expect(res.rawPayload).to.eql(bigFile) + expect(res.headers['content-length']).to.equal(res.rawPayload.length).to.equal(15000000) expect(res.headers['x-ipfs-path']).to.equal(`/ipfs/${bigFileHash}`) expect(res.headers['etag']).to.equal(`"${bigFileHash}"`) + expect(res.headers['last-modified']).to.equal('Thu, 01 Jan 1970 00:00:01 GMT') expect(res.headers['cache-control']).to.equal('public, max-age=29030400, immutable') expect(res.headers['content-type']).to.equal('application/octet-stream') }) + it('load specific byte range of a file (from-)', async () => { + // use 12 byte text file to make it easier to debug ;-) + const fileCid = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + const fileLength = 12 + const range = { from: 1, length: 11 } + + // get full file first to read accept-ranges and etag headers + const resFull = await gateway.inject({ + method: 'GET', + url: '/ipfs/' + fileCid + }) + expect(resFull.statusCode).to.equal(200) + expect(resFull.headers['accept-ranges']).to.equal('bytes') + expect(resFull.headers['etag']).to.equal(`"${fileCid}"`) + expect(resFull.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + expect(resFull.headers['content-length']).to.equal(resFull.rawPayload.length).to.equal(fileLength) + + // extract expected chunk of interest + const rangeValue = `bytes=${range.from}-` + const expectedChunk = resFull.rawPayload.slice(range.from) + + // const expectedChunkBytes = bigFile.slice(range.from, range.to) + const resRange = await gateway.inject({ + method: 'GET', + url: '/ipfs/' + fileCid, + headers: { + 'range': rangeValue, + 'if-range': resFull.headers.etag // if-range is meaningless for immutable /ipfs/, but will matter for /ipns/ + } + }) + + // range headers + expect(resRange.statusCode).to.equal(206) + expect(resRange.headers['content-range']).to.equal(`bytes ${range.from}-${range.length}/${fileLength}`) + expect(resRange.headers['content-length']).to.equal(resRange.rawPayload.length).to.equal(range.length) + expect(resRange.headers['accept-ranges']).to.equal(undefined) + expect(resRange.rawPayload).to.deep.equal(expectedChunk) + // regular headers that should also be present + expect(resRange.headers['x-ipfs-path']).to.equal(`/ipfs/${fileCid}`) + expect(resRange.headers['etag']).to.equal(`"${fileCid}"`) + expect(resRange.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + expect(resRange.headers['last-modified']).to.equal('Thu, 01 Jan 1970 00:00:01 GMT') + expect(resRange.headers['content-type']).to.equal('application/octet-stream') + }) + + it('load specific byte range of a file (from-to)', async () => { + // use 12 byte text file to make it easier to debug ;-) + const fileCid = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + const fileLength = 12 + const range = { from: 1, to: 3, length: 3 } + + // get full file first to read accept-ranges and etag headers + const resFull = await gateway.inject({ + method: 'GET', + url: '/ipfs/' + fileCid + }) + expect(resFull.statusCode).to.equal(200) + expect(resFull.headers['accept-ranges']).to.equal('bytes') + expect(resFull.headers['etag']).to.equal(`"${fileCid}"`) + expect(resFull.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + expect(resFull.headers['content-length']).to.equal(resFull.rawPayload.length).to.equal(fileLength) + + // extract expected chunk of interest + const rangeValue = `bytes=${range.from}-${range.to}` + const expectedChunk = resFull.rawPayload.slice(range.from, range.to + 1) // include end + + // const expectedChunkBytes = bigFile.slice(range.from, range.to) + const resRange = await gateway.inject({ + method: 'GET', + url: '/ipfs/' + fileCid, + headers: { + 'range': rangeValue, + 'if-range': resFull.headers.etag // if-range is meaningless for immutable /ipfs/, but will matter for /ipns/ + } + }) + + // range headers + expect(resRange.statusCode).to.equal(206) + expect(resRange.headers['content-range']).to.equal(`bytes ${range.from}-${range.to}/${fileLength}`) + expect(resRange.headers['content-length']).to.equal(resRange.rawPayload.length).to.equal(range.length) + expect(resRange.headers['accept-ranges']).to.equal(undefined) + expect(resRange.rawPayload).to.deep.equal(expectedChunk) + // regular headers that should also be present + expect(resRange.headers['x-ipfs-path']).to.equal(`/ipfs/${fileCid}`) + expect(resRange.headers['etag']).to.equal(`"${fileCid}"`) + expect(resRange.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + expect(resRange.headers['last-modified']).to.equal('Thu, 01 Jan 1970 00:00:01 GMT') + expect(resRange.headers['content-type']).to.equal('application/octet-stream') + }) + + // This one is tricky, as "-to" does not mean implicit "0-to", + // but "give me last N bytes" + // More at https://tools.ietf.org/html/rfc7233#section-2.1 + it('load specific byte range of a file (-tail AKA bytes from end)', async () => { + // use 12 byte text file to make it easier to debug ;-) + const fileCid = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + const fileLength = 12 + const range = { tail: 7, from: 5, to: 11, length: 7 } + + // get full file first to read accept-ranges and etag headers + const resFull = await gateway.inject({ + method: 'GET', + url: '/ipfs/' + fileCid + }) + expect(resFull.statusCode).to.equal(200) + expect(resFull.headers['accept-ranges']).to.equal('bytes') + expect(resFull.headers['etag']).to.equal(`"${fileCid}"`) + expect(resFull.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + expect(resFull.headers['content-length']).to.equal(resFull.rawPayload.length).to.equal(fileLength) + + // extract expected chunk of interest + const rangeValue = `bytes=-${range.tail}` + const expectedChunk = resFull.rawPayload.slice(range.from, range.to + 1) // include end + + // const expectedChunkBytes = resFull.rawPayload.slice(range.from, range.to) + const resRange = await gateway.inject({ + method: 'GET', + url: '/ipfs/' + fileCid, + headers: { + 'range': rangeValue, + 'if-range': resFull.headers.etag // if-range is meaningless for immutable /ipfs/, but will matter for /ipns/ + } + }) + + // range headers + expect(resRange.statusCode).to.equal(206) + expect(resRange.headers['content-range']).to.equal(`bytes ${range.from}-${range.to}/${fileLength}`) + expect(resRange.headers['content-length']).to.equal(resRange.rawPayload.length).to.equal(range.length) + expect(resRange.headers['accept-ranges']).to.equal(undefined) + expect(resRange.rawPayload).to.deep.equal(expectedChunk) + // regular headers that should also be present + expect(resRange.headers['x-ipfs-path']).to.equal(`/ipfs/${fileCid}`) + expect(resRange.headers['etag']).to.equal(`"${fileCid}"`) + expect(resRange.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + expect(resRange.headers['last-modified']).to.equal('Thu, 01 Jan 1970 00:00:01 GMT') + expect(resRange.headers['content-type']).to.equal('application/octet-stream') + }) + + it('return 416 (Range Not Satisfiable) on invalid range request', async () => { + // use 12 byte text file to make it easier to debug ;-) + const fileCid = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + // requesting range outside of file length + const rangeValue = 'bytes=42-100' + + // const expectedChunkBytes = bigFile.slice(range.from, range.to) + const resRange = await gateway.inject({ + method: 'GET', + url: '/ipfs/' + fileCid, + headers: { 'range': rangeValue } + }) + + // Expect 416 Range Not Satisfiable + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/416 + expect(resRange.statusCode).to.equal(416) + expect(resRange.headers['content-range']).to.equal('bytes */12') + expect(resRange.headers['cache-control']).to.equal('no-cache') + }) + it('load a jpg file', async () => { const kitty = 'QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ/cat.jpg' const kittyDirectCid = 'Qmd286K6pohQcTKYqnS1YhWrCiS4gz7Xi34sdwMe9USZ7u' @@ -173,9 +401,11 @@ describe('HTTP Gateway', function () { expect(res.statusCode).to.equal(200) expect(res.headers['content-type']).to.equal('image/jpeg') + expect(res.headers['content-length']).to.equal(res.rawPayload.length).to.equal(443230) expect(res.headers['x-ipfs-path']).to.equal('/ipfs/' + kitty) expect(res.headers['etag']).to.equal(`"${kittyDirectCid}"`) expect(res.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + expect(res.headers['last-modified']).to.equal('Thu, 01 Jan 1970 00:00:01 GMT') expect(res.headers.etag).to.equal('"Qmd286K6pohQcTKYqnS1YhWrCiS4gz7Xi34sdwMe9USZ7u"') expect(res.headers.suborigin).to.equal('ipfs000bafybeidsg6t7ici2osxjkukisd5inixiunqdpq2q5jy4a2ruzdf6ewsqk4') @@ -220,6 +450,8 @@ describe('HTTP Gateway', function () { expect(res.headers['content-type']).to.equal('text/html; charset=utf-8') expect(res.headers['x-ipfs-path']).to.equal('/ipfs/' + dir) expect(res.headers['cache-control']).to.equal('no-cache') + expect(res.headers['last-modified']).to.equal('Thu, 01 Jan 1970 00:00:01 GMT') + expect(res.headers['content-length']).to.equal(res.rawPayload.length) expect(res.headers.etag).to.equal(undefined) expect(res.headers.suborigin).to.equal('ipfs000bafybeidsg6t7ici2osxjkukisd5inixiunqdpq2q5jy4a2ruzdf6ewsqk4') @@ -241,6 +473,8 @@ describe('HTTP Gateway', function () { expect(res.headers['content-type']).to.equal('text/html; charset=utf-8') expect(res.headers['x-ipfs-path']).to.equal('/ipfs/' + dir) expect(res.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + expect(res.headers['last-modified']).to.equal('Thu, 01 Jan 1970 00:00:01 GMT') + expect(res.headers['content-length']).to.equal(res.rawPayload.length) expect(res.headers.etag).to.equal('"Qma6665X5k3zti8nKy7gmXK2BndNDSkgmANpV6k3FUjUeg"') expect(res.headers.suborigin).to.equal('ipfs000bafybeigccfheqv7upr4k64bkg5b5wiwelunyn2l2rbirmm43m34lcpuqqe') expect(res.rawPayload).to.deep.equal(directoryContent['index.html']) @@ -258,6 +492,8 @@ describe('HTTP Gateway', function () { expect(res.headers['content-type']).to.equal('text/html; charset=utf-8') expect(res.headers['x-ipfs-path']).to.equal('/ipfs/' + dir) expect(res.headers['cache-control']).to.equal('public, max-age=29030400, immutable') + expect(res.headers['last-modified']).to.equal('Thu, 01 Jan 1970 00:00:01 GMT') + expect(res.headers['content-length']).to.equal(res.rawPayload.length) expect(res.headers.etag).to.equal('"QmUBKGqJWiJYMrNed4bKsbo1nGYGmY418WCc2HgcwRvmHc"') expect(res.headers.suborigin).to.equal('ipfs000bafybeigccfheqv7upr4k64bkg5b5wiwelunyn2l2rbirmm43m34lcpuqqe') expect(res.rawPayload).to.deep.equal(directoryContent['nested-folder/nested.html']) @@ -276,6 +512,7 @@ describe('HTTP Gateway', function () { expect(res.headers['x-ipfs-path']).to.equal(undefined) }) + // TODO: check if interop for this exists and if not, match behavior of go-ipfs it('redirect to webpage index.html', async () => { const dir = 'QmbQD7EMEL1zeebwBsWEfA3ndgSS6F7S6iTuwuqasPgVRi/' From 7e237fc927d4c4bea8f25c3f571da3375d6a9796 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 8 May 2019 09:10:18 +0100 Subject: [PATCH 3/3] refactor: remove redundant try/catch License: MIT Signed-off-by: Alan Shaw --- src/http/gateway/resources/gateway.js | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/http/gateway/resources/gateway.js b/src/http/gateway/resources/gateway.js index 9b903625ac..fda0610287 100644 --- a/src/http/gateway/resources/gateway.js +++ b/src/http/gateway/resources/gateway.js @@ -143,23 +143,19 @@ module.exports = { const responseStream = new ResponseStream() // Pass-through Content-Type sniffing over initial bytes - const contentType = await new Promise((resolve, reject) => { - try { - const peekBytes = fileType.minimumBytes - peek(rawStream, peekBytes, (err, streamHead, outputStream) => { - if (err) { - log.error(err) - return reject(err) - } - outputStream.pipe(responseStream) - resolve(detectContentType(ref, streamHead)) - }) - } catch (err) { - log.error(err) - reject(err) - } + const { peekedStream, contentType } = await new Promise((resolve, reject) => { + const peekBytes = fileType.minimumBytes + peek(rawStream, peekBytes, (err, streamHead, peekedStream) => { + if (err) { + log.error(err) + return reject(err) + } + resolve({ peekedStream, contentType: detectContentType(ref, streamHead) }) + }) }) + peekedStream.pipe(responseStream) + const res = h.response(responseStream).code(rangeResponse ? 206 : 200) // Etag maps directly to an identifier for a specific version of a resource