implement more caching & public cache api

http/vibe/http/fileserver.d

@@ -16,11 +16,15 @@ import vibe.inet.mimetypes;
 import vibe.inet.url;
 import vibe.internal.interfaceproxy;
 
+import std.ascii : isWhite;
+import std.algorithm;
 import std.conv;
 import std.datetime;
 import std.digest.md;
+import std.exception;
+import std.range : popFront, empty, drop;
 import std.string;
-import std.algorithm;
+import std.typecons : Flag, Yes, No;
 
 @safe:
 
@@ -164,12 +168,12 @@ HTTPServerRequestDelegateS serveStaticFile(string local_path, HTTPFileServerSett
 	When serving a file, certain request headers are supported to avoid sending
 	the file if the client has it already cached. These headers are
 	`"If-Modified-Since"` and `"If-None-Match"`. The client will be delivered
-	with the necessary `"Etag"` (generated from the path, size and last
-	modification time of the file) and `"Last-Modified"` headers.
+	with the necessary `"Etag"` (generated from size and last modification time
+	of the file) and `"Last-Modified"` headers.
 
-	The cache control directives `"Expires"` and `"Cache-Control"` will also be
+	The cache control directives `"Expires"` and/or `"Cache-Control"` will also be
 	emitted if the `HTTPFileServerSettings.maxAge` field is set to a positive
-	duration.
+	duration and/or `HTTPFileServerSettings.cacheControl` has been set.
 
 	Finally, HEAD requests will automatically be handled without reading the
 	actual file contents. Am empty response body is written instead.
@@ -203,6 +207,20 @@ class HTTPFileServerSettings {
 	/// Maximum cache age to report to the client (zero by default)
 	Duration maxAge = 0.seconds;
 
+	/** Cache control to control where cache can be saved, if at all, such as
+		proxies, the storage, etc.
+
+		Leave null or empty to not emit any cache control directives other than
+		max-age if maxAge is set.
+
+		Common values include: public for making a shared resource cachable across
+		multiple users or private for a response that should only be cached for a
+		single user.
+
+		See https://developer.mozilla.org/de/docs/Web/HTTP/Headers/Cache-Control
+	*/
+	string cacheControl = null;
+
 	/// General options
 	HTTPFileServerOption options = HTTPFileServerOption.defaults; /// additional options
 
@@ -282,32 +300,8 @@ private void sendFileImpl(scope HTTPServerRequest req, scope HTTPServerResponse
 		return;
 	}
 
-	auto lastModified = toRFC822DateTimeString(dirent.timeModified.toUTC());
-	// simple etag generation
-	auto etag = "\"" ~ hexDigest!MD5(pathstr ~ ":" ~ lastModified ~ ":" ~ to!string(dirent.size)).idup ~ "\"";
-
-	res.headers["Last-Modified"] = lastModified;
-	res.headers["Etag"] = etag;
-	if (settings.maxAge > seconds(0)) {
-		auto expireTime = Clock.currTime(UTC()) + settings.maxAge;
-		res.headers["Expires"] = toRFC822DateTimeString(expireTime);
-		res.headers["Cache-Control"] = "max-age="~to!string(settings.maxAge.total!"seconds");
-	}
-
-	if( auto pv = "If-Modified-Since" in req.headers ) {
-		if( *pv == lastModified ) {
-			res.statusCode = HTTPStatus.NotModified;
-			res.writeVoidBody();
-			return;
-		}
-	}
-
-	if( auto pv = "If-None-Match" in req.headers ) {
-		if ( *pv == etag ) {
-			res.statusCode = HTTPStatus.NotModified;
-			res.writeVoidBody();
-			return;
-		}
+	if (handleCacheFile(req, res, dirent, settings.cacheControl, settings.maxAge)) {
+		return;
 	}
 
 	auto mimetype = res.headers.get("Content-Type", getMimeTypeForFile(pathstr));
@@ -427,3 +421,292 @@ private void sendFileImpl(scope HTTPServerRequest req, scope HTTPServerResponse
 		logTrace("sent file %d, %s!", fil.size, res.headers["Content-Type"]);
 	}
 }
+
+/**
+	Calls $(D handleCache) with prefilled etag and lastModified value based on a file.
+
+	See_Also: handleCache
+
+	Returns: $(D true) if the cache was already handled and no further response must be sent or $(D false) if a response must be sent.
+*/
+bool handleCacheFile(scope HTTPServerRequest req, scope HTTPServerResponse res,
+		string file, string cache_control = null, Duration max_age = Duration.zero)
+{
+	return handleCacheFile(req, res, NativePath(file), cache_control, max_age);
+}
+
+/// ditto
+bool handleCacheFile(scope HTTPServerRequest req, scope HTTPServerResponse res,
+		NativePath file, string cache_control = null, Duration max_age = Duration.zero)
+{
+	if (!existsFile(file)) {
+		return false;
+	}
+
+	FileInfo ent;
+	try {
+		ent = getFileInfo(file);
+	} catch (Exception) {
+		throw new HTTPStatusException(HTTPStatus.internalServerError,
+				"Failed to get information for the file due to a file system error.");
+	}
+
+	return handleCacheFile(req, res, ent, cache_control, max_age);
+}
+
+/// ditto
+bool handleCacheFile(scope HTTPServerRequest req, scope HTTPServerResponse res,
+		FileInfo dirent, string cache_control = null, Duration max_age = Duration.zero)
+{
+	import std.bitmanip : nativeToLittleEndian;
+	import std.digest.md : MD5, toHexString;
+
+	SysTime lastModified = dirent.timeModified;
+	const weak = cast(Flag!"weak") dirent.isDirectory;
+	auto etag = ETag.md5(weak, lastModified.stdTime.nativeToLittleEndian, dirent.size.nativeToLittleEndian);
+
+	return handleCache(req, res, etag, lastModified, cache_control, max_age);
+}
+
+/**
+	Processes header tags in a request and writes responses given on requested cache status.
+
+	Params:
+		req = the client request used to determine cache control flow.
+		res = the response to write cache headers to.
+		etag = if set to anything except .init, adds a Etag header to the response and enables handling of If-Match and If-None-Match cache control request headers.
+		last_modified = if set to anything except .init, adds a Last-Modified header to the response and enables handling of If-Modified-Since and If-Unmodified-Since cache control request headers.
+		cache_control = if set, adds or modifies the Cache-Control header in the response to this string. Might get an additional max-age value appended if max_age is set.
+		max_age = optional duration to set the Expires header and Cache-Control max-age part to. (if no existing `max-age=` part is given in the cache_control parameter)
+
+	Returns: $(D true) if the cache was already handled and no further response must be sent or $(D false) if a response must be sent.
+*/
+bool handleCache(scope HTTPServerRequest req, scope HTTPServerResponse res, ETag etag,
+		SysTime last_modified, string cache_control = null, Duration max_age = Duration.zero)
+{
+	// https://tools.ietf.org/html/rfc7232#section-4.1
+	// and
+	// https://tools.ietf.org/html/rfc7232#section-6
+	string lastModifiedString;
+	if (last_modified != SysTime.init) {
+		lastModifiedString = toRFC822DateTimeString(last_modified);
+		res.headers["Last-Modified"] = lastModifiedString;
+	}
+
+	if (etag != ETag.init) {
+		res.headers["Etag"] = etag.toString;
+	}
+
+	if (max_age > Duration.zero) {
+		res.headers["Expires"] = toRFC822DateTimeString(Clock.currTime(UTC()) + max_age);
+	}
+
+	if (cache_control.length) {
+		if (max_age > Duration.zero && !cache_control.canFind("max-age=")) {
+			res.headers["Cache-Control"] = cache_control
+				~ ", max-age=" ~ to!string(max_age.total!"seconds");
+		} else {
+			res.headers["Cache-Control"] = cache_control;
+		}
+	} else if (max_age > Duration.zero) {
+		res.headers["Cache-Control"] = text("max-age=", max_age.total!"seconds");
+	}
+
+	// https://tools.ietf.org/html/rfc7232#section-3.1
+	string ifMatch = req.headers.get("If-Match");
+	if (ifMatch.length) {
+		if (!cacheMatch(ifMatch, etag, No.allowWeak)) {
+			res.statusCode = HTTPStatus.preconditionFailed;
+			res.writeVoidBody();
+			return true;
+		}
+	}
+	else if (last_modified != SysTime.init) {
+		// https://tools.ietf.org/html/rfc7232#section-3.4
+		string ifUnmodifiedSince = req.headers.get("If-Unmodified-Since");
+		if (ifUnmodifiedSince.length) {
+			const check = lastModifiedString != ifUnmodifiedSince
+				|| last_modified > parseRFC822DateTimeString(ifUnmodifiedSince);
+			if (check) {
+				res.statusCode = HTTPStatus.preconditionFailed;
+				res.writeVoidBody();
+				return true;
+			}
+		}
+	}
+
+	// https://tools.ietf.org/html/rfc7232#section-3.2
+	string ifNoneMatch = req.headers.get("If-None-Match");
+	if (ifNoneMatch.length) {
+		if (cacheMatch(ifNoneMatch, etag, Yes.allowWeak)) {
+			if (req.method.among!(HTTPMethod.GET, HTTPMethod.HEAD))
+				res.statusCode = HTTPStatus.notModified;
+			else
+				res.statusCode = HTTPStatus.preconditionFailed;
+			res.writeVoidBody();
+			return true;
+		}
+	}
+	else if (last_modified != SysTime.init && req.method.among!(HTTPMethod.GET, HTTPMethod.HEAD)) {
+		// https://tools.ietf.org/html/rfc7232#section-3.3
+		string ifModifiedSince = req.headers.get("If-Modified-Since");
+		if (ifModifiedSince.length) {
+			const check = lastModifiedString == ifModifiedSince ||
+				last_modified <= parseRFC822DateTimeString(ifModifiedSince);
+			if (check) {
+				res.statusCode = HTTPStatus.notModified;
+				res.writeVoidBody();
+				return true;
+			}
+		}
+	}
+
+	// TODO: support If-Range here
+
+	return false;
+}
+
+/**
+	Represents an Entity-Tag value for use inside HTTP Cache headers.
+
+	Standards: https://tools.ietf.org/html/rfc7232#section-2.3
+*/
+struct ETag
+{
+	bool weak;
+	string tag;
+
+	static ETag parse(string s)
+	{
+		enforce!ConvException(s.endsWith('"'));
+
+		if (s.startsWith(`W/"`)) {
+			ETag ret = { weak: true, tag: s[3 .. $ - 1] };
+			return ret;
+		} else if (s.startsWith('"')) {
+			ETag ret;
+			ret.tag = s[1 .. $ - 1];
+			return ret;
+		} else {
+			throw new ConvException(`ETag didn't start with W/" nor with " !`);
+		}
+	}
+
+	string toString() const @property
+	{
+		return text(weak ? `W/"` : `"`, tag, '"');
+	}
+
+	/**
+		Encodes the bytes with URL Base64 to a human readable string and returns an ETag struct wrapping it.
+	 */
+	static ETag fromBytesBase64URLNoPadding(scope const(ubyte)[] bytes, Flag!"weak" weak = No.weak)
+	{
+		import std.base64 : Base64URLNoPadding;
+
+		return ETag(weak, Base64URLNoPadding.encode(bytes).idup);
+	}
+
+	/**
+		Hashes the input bytes with md5 and returns an URL Base64 encoded representation as ETag.
+	 */
+	static ETag md5(T...)(Flag!"weak" weak, T data)
+	{
+		import std.digest.md : md5Of;
+
+		return fromBytesBase64URLNoPadding(md5Of(data), weak);
+	}
+}
+
+/**
+	Matches a given match expression with a specific ETag. Can allow or disallow weak ETags and supports multiple tags.
+
+	Standards: https://tools.ietf.org/html/rfc7232#section-2.3.2
+*/
+bool cacheMatch(string match, ETag etag, Flag!"allowWeak" allow_weak)
+{
+	if (match == "*") {
+		return true;
+	}
+
+	if ((etag.weak && !allow_weak) || !match.length) {
+		return false;
+	}
+
+	auto allBytes = match.representation;
+	auto range = allBytes;
+
+	while (!range.empty)
+	{
+		range = range.stripLeft!isWhite;
+		bool isWeak = range.skipOver("W/");
+		if (!range.skipOver('"'))
+			return false; // malformed
+
+		auto end = range.countUntil('"');
+		if (end == -1)
+			return false; // malformed
+
+		const check = range[0 .. end];
+		range = range[end .. $];
+
+		if (allow_weak || !isWeak) {
+			if (check == etag.tag) {
+				return true;
+			}
+		}
+
+		range.skipOver('"');
+		range = range.stripLeft!isWhite;
+
+		if (!range.skipOver(","))
+			return false; // malformed
+	}
+
+	return false;
+}
+
+unittest
+{
+	// from RFC 7232 Section 2.3.2
+	// +--------+--------+-------------------+-----------------+
+	// | ETag 1 | ETag 2 | Strong Comparison | Weak Comparison |
+	// +--------+--------+-------------------+-----------------+
+	// | W/"1"  | W/"1"  | no match          | match           |
+	// | W/"1"  | W/"2"  | no match          | no match        |
+	// | W/"1"  | "1"    | no match          | match           |
+	// | "1"    | "1"    | match             | match           |
+	// +--------+--------+-------------------+-----------------+
+
+	assert(!cacheMatch(`W/"1"`, ETag(Yes.weak, "1"), No.allowWeak));
+	assert( cacheMatch(`W/"1"`, ETag(Yes.weak, "1"), Yes.allowWeak));
+
+	assert(!cacheMatch(`W/"1"`, ETag(Yes.weak, "2"), No.allowWeak));
+	assert(!cacheMatch(`W/"1"`, ETag(Yes.weak, "2"), Yes.allowWeak));
+
+	assert(!cacheMatch(`W/"1"`, ETag(No.weak, "1"), No.allowWeak));
+	assert( cacheMatch(`W/"1"`, ETag(No.weak, "1"), Yes.allowWeak));
+
+	assert(cacheMatch(`"1"`, ETag(No.weak, "1"), No.allowWeak));
+	assert(cacheMatch(`"1"`, ETag(No.weak, "1"), Yes.allowWeak));
+
+	assert(cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "xyzzy"), No.allowWeak));
+	assert(cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "xyzzy"), Yes.allowWeak));
+
+	assert(!cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "xyzzz"), No.allowWeak));
+	assert(!cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "xyzzz"), Yes.allowWeak));
+
+	assert(cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "r2d2xxxx"), No.allowWeak));
+	assert(cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "r2d2xxxx"), Yes.allowWeak));
+
+	assert(cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "c3piozzzz"), No.allowWeak));
+	assert(cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "c3piozzzz"), Yes.allowWeak));
+
+	assert(!cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, ""), No.allowWeak));
+	assert(!cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, ""), Yes.allowWeak));
+
+	assert(!cacheMatch(`"xyzzy",W/"r2d2xxxx", "c3piozzzz"`, ETag(Yes.weak, "r2d2xxxx"), No.allowWeak));
+	assert( cacheMatch(`"xyzzy",W/"r2d2xxxx", "c3piozzzz"`, ETag(Yes.weak, "r2d2xxxx"), Yes.allowWeak));
+	assert(!cacheMatch(`"xyzzy",W/"r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "r2d2xxxx"), No.allowWeak));
+	assert( cacheMatch(`"xyzzy",W/"r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "r2d2xxxx"), Yes.allowWeak));
+}