1 module fileCache; 2 3 import vibe.core.file; 4 import vibe.core.log; 5 import vibe.http.server; 6 import vibe.inet.message; 7 import vibe.inet.mimetypes; 8 import vibe.inet.url; 9 import vibe.stream.memory; 10 11 import std.conv; 12 import std.datetime; 13 import std.digest.md; 14 import std.string; 15 import std.regex; 16 17 class CacheEntry 18 { 19 FileInfo dirent; 20 string pathstr; 21 string lastModified; 22 string etag; 23 string cacheControl; 24 string contentType; 25 string contentLength; 26 bool notFound; 27 bool contentsTooLarge; 28 bool isCompressedContent; 29 ubyte[] content; 30 } 31 32 // Max 3MB 33 immutable uint DEFAULT_CACHE_SIZE = 3*1024*1024; 34 private CacheEntry[string] entries; 35 36 class FileServerCache 37 { 38 private CacheEntry[string] entries; 39 private uint remainingSize = DEFAULT_CACHE_SIZE; 40 private uint maxSize = DEFAULT_CACHE_SIZE; 41 private uint myMaxItemSize; 42 43 @property uint maxItemSize() { return myMaxItemSize; } 44 45 this(uint maxSize = DEFAULT_CACHE_SIZE, uint maxItemSize = DEFAULT_CACHE_SIZE) 46 { 47 this.maxSize = maxSize; 48 this.myMaxItemSize = maxItemSize; 49 } 50 51 void tryPut(CacheEntry info) 52 { 53 auto addSize = info.dirent.size; 54 if(addSize > myMaxItemSize) return; 55 if(addSize > remainingSize) return; 56 57 // TODO: Replacement policy 58 remainingSize -= addSize; 59 entries[info.pathstr] = info; 60 } 61 62 CacheEntry get(string pathstr) 63 { 64 if(auto pv = pathstr in entries) 65 { 66 return *pv; 67 } 68 69 return null; 70 } 71 } 72 73 auto byteRangeExpression = ctRegex!(`^bytes=(\d+)?-(\d+)?$`); 74 75 /** 76 Additional options for the static file server. 77 */ 78 enum HTTPFileServerOption { 79 none = 0, 80 /// respond with 404 if a file was not found 81 failIfNotFound = 1 << 0, 82 /// serve index.html for directories 83 serveIndexHTML = 1 << 1, 84 /// default options are serveIndexHTML 85 defaults = serveIndexHTML, 86 } 87 88 /** 89 Configuration options for the static file server. 90 */ 91 class HTTPFileServerSettings { 92 /// Prefix of the request path to strip before looking up files 93 string serverPathPrefix = "/"; 94 95 /// Maximum cache age to report to the client (24 hours by default) 96 Duration maxAge;// = hours(24); 97 98 /// General options 99 HTTPFileServerOption options = HTTPFileServerOption.defaults; /// additional options 100 101 /// File cache 102 FileServerCache cache = null; 103 104 /** 105 Called just before headers and data are sent. 106 Allows headers to be customized, or other custom processing to be performed. 107 108 Note: Any changes you make to the response, physicalPath, or anything 109 else during this function will NOT be verified by Vibe.d for correctness. 110 Make sure any alterations you make are complete and correct according to HTTP spec. 111 */ 112 void delegate(scope HTTPServerRequest req, scope HTTPServerResponse res, ref string physicalPath) preWriteCallback = null; 113 114 this() 115 { 116 // need to use the contructor because the Ubuntu 13.10 GDC cannot CTFE dur() 117 maxAge = 0.seconds; 118 } 119 120 this(string path_prefix) 121 { 122 this(); 123 serverPathPrefix = path_prefix; 124 } 125 } 126 127 private CacheEntry prepareRequestResponseInfo(string pathstr, HTTPFileServerSettings settings) 128 { 129 auto info = new CacheEntry; 130 info.pathstr = pathstr; 131 132 // return if the file does not exist 133 if (!existsFile(pathstr)){ 134 info.notFound = true; 135 return info; 136 } 137 138 FileInfo dirent; 139 try dirent = getFileInfo(pathstr); 140 catch(Exception){ 141 throw new HTTPStatusException(HTTPStatus.InternalServerError, "Failed to get information for the file due to a file system error."); 142 } 143 144 if (dirent.isDirectory) { 145 if (settings.options & HTTPFileServerOption.serveIndexHTML) 146 return prepareRequestResponseInfo(pathstr ~ "index.html", settings); 147 logDebugV("Hit directory when serving files, ignoring: %s", pathstr); 148 info.notFound = true; 149 return info; 150 } 151 152 info.dirent = dirent; 153 154 info.lastModified = toRFC822DateTimeString(dirent.timeModified.toUTC()); 155 // simple etag generation 156 info.etag = "\"" ~ hexDigest!MD5(pathstr ~ ":" ~ info.lastModified ~ ":" ~ to!string(dirent.size)).idup ~ "\""; 157 158 info.contentType = getMimeTypeForFile(pathstr); 159 info.contentLength = to!string(dirent.size); 160 161 info.contentsTooLarge = info.dirent.size > (settings.cache !is null ? settings.cache.maxItemSize : DEFAULT_CACHE_SIZE); 162 if(info.contentsTooLarge) return info; 163 164 FileStream fil; 165 try { 166 fil = openFile(info.pathstr); 167 info.content.length = cast(int)info.dirent.size; 168 fil.read(info.content); 169 } catch( Exception e ){ 170 // TODO: handle non-existant files differently than locked files? 171 logDebug("Failed to open file %s: %s", info.pathstr, e.toString()); 172 info.notFound = true; 173 } 174 finally { 175 fil.close(); 176 } 177 178 return info; 179 } 180 181 private void writeBody(CacheEntry info, ulong begin, HTTPServerResponse res) 182 { 183 if(info.contentsTooLarge) 184 { 185 auto fil = openFile(info.pathstr); 186 scope(exit) fil.close(); 187 res.writeBody(fil); 188 } 189 else if(begin == 0) 190 { 191 res.writeBody(info.content); 192 } 193 else 194 { 195 auto stream = new MemoryStream(info.content); 196 stream.seek(begin); 197 res.writeRawBody(stream); 198 } 199 } 200 201 private void sendFileCacheImpl(scope HTTPServerRequest req, scope HTTPServerResponse res, Path path, HTTPFileServerSettings settings) 202 { 203 CacheEntry info; 204 auto pathstr = path.toNativeString(); 205 206 if(settings.cache is null) 207 { 208 info = prepareRequestResponseInfo(pathstr, settings); 209 } else { 210 info = settings.cache.get(pathstr); 211 if(info is null) { 212 info = prepareRequestResponseInfo(pathstr, settings); 213 settings.cache.tryPut(info); 214 } 215 } 216 217 if(info.notFound) 218 { 219 if(settings.options & HTTPFileServerOption.failIfNotFound) throw new HTTPStatusException(HTTPStatus.NotFound); 220 res.statusCode = HTTPStatus.notFound; 221 return; 222 } 223 224 res.headers["Accept-Ranges"] = "bytes"; 225 226 res.headers["Last-Modified"] = info.lastModified; 227 res.headers["Etag"] = info.etag; 228 229 if (settings.maxAge > seconds(0)) { 230 auto expireTime = Clock.currTime(UTC()) + settings.maxAge; 231 res.headers["Expires"] = toRFC822DateTimeString(expireTime); 232 res.headers["Cache-Control"] = "max-age=" ~ to!string(settings.maxAge.total!"seconds"); 233 } 234 235 if( auto pv = "If-Modified-Since" in req.headers ) { 236 if( *pv == info.lastModified ) { 237 res.statusCode = HTTPStatus.NotModified; 238 res.writeVoidBody(); 239 return; 240 } 241 } 242 243 if( auto pv = "If-None-Match" in req.headers ) { 244 if ( *pv == info.etag ) { 245 res.statusCode = HTTPStatus.NotModified; 246 res.writeVoidBody(); 247 return; 248 } 249 } 250 251 if( auto pv = "If-Unmodified-Since" in req.headers ) { 252 if( *pv != info.lastModified ) { 253 res.statusCode = HTTPStatus.NotModified; 254 res.writeVoidBody(); 255 return; 256 } 257 } 258 259 if( auto pv = "If-Match" in req.headers ) { 260 if ( *pv != info.etag ) { 261 res.statusCode = HTTPStatus.NotModified; 262 res.writeVoidBody(); 263 return; 264 } 265 } 266 267 res.headers["Content-Type"] = info.contentType; 268 269 if( auto pv = "Range" in req.headers ) 270 { 271 auto match = matchFirst(*pv, byteRangeExpression); 272 // both inclusive indices 273 uint begin, end; 274 auto p1 = match.length > 1 && match[1] != ""; 275 auto p2 = match.length > 2 && match[2] != ""; 276 277 if(p1 || p2) 278 { 279 if ("Content-Encoding" in res.headers) 280 res.headers.remove("Content-Encoding"); 281 282 if(p1 && !p2) 283 { 284 begin = to!uint(match[1]); 285 end = cast(uint)info.dirent.size - 1; 286 } 287 else if(!p1 && p2) 288 { 289 begin = cast(uint)info.dirent.size - to!uint(match[2]); 290 end = cast(uint)info.dirent.size - 1; 291 } 292 else if(p1 && p2) 293 { 294 begin = to!uint(match[1]); 295 end = to!uint(match[2]); 296 } 297 298 res.statusCode = HTTPStatus.partialContent; 299 res.headers["Content-Range"] = "bytes " ~ to!string(begin) ~ "-" ~ to!string(end) ~ "/" ~ info.contentLength; 300 301 auto length = end - begin + 1; 302 res.headers["Content-Length"] = to!string(length); 303 304 if(info.contentsTooLarge) 305 { 306 auto fil = openFile(info.pathstr); 307 scope(exit) fil.close(); 308 fil.seek(begin); 309 res.writeRawBody(fil); 310 } 311 else 312 { 313 auto stream = new MemoryStream(info.content[begin..end+1]); 314 scope(exit) delete stream; 315 res.writeRawBody(stream); 316 } 317 318 logTrace("sent partial file %d-%d, %s!", begin, end, res.headers["Content-Type"]); 319 return; 320 } 321 } 322 323 // avoid double-compression 324 if ("Content-Encoding" in res.headers && isCompressedFormat(info.contentType)) 325 res.headers.remove("Content-Encoding"); 326 327 res.headers["Content-Length"] = info.contentLength; 328 329 if(settings.preWriteCallback) 330 { 331 settings.preWriteCallback(req, res, pathstr); 332 } 333 334 // for HEAD responses, stop here 335 if( res.isHeadResponse() ){ 336 res.writeVoidBody(); 337 assert(res.headerWritten); 338 logDebug("sent file header %d, %s!", info.dirent.size, res.headers["Content-Type"]); 339 return; 340 } 341 342 if(info.contentsTooLarge) 343 { 344 auto fil = openFile(info.pathstr); 345 scope(exit) fil.close(); 346 res.writeBody(fil); 347 } 348 else 349 { 350 res.writeBody(info.content); 351 } 352 353 logTrace("sent file %d, %s!", info.dirent.size, res.headers["Content-Type"]); 354 } 355 356 357 358 /** 359 Returns a request handler that serves files from the specified directory. 360 361 See `sendFile` for more information. 362 363 Params: 364 local_path = Path to the folder to serve files from. 365 settings = Optional settings object enabling customization of how 366 the files get served. 367 368 Returns: 369 A request delegate is returned, which is suitable for registering in 370 a `URLRouter` or for passing to `listenHTTP`. 371 372 See_Also: `serveStaticFile`, `sendFile` 373 */ 374 HTTPServerRequestDelegateS serveStaticFiles(Path local_path, HTTPFileServerSettings settings = null) 375 { 376 if (!settings) settings = new HTTPFileServerSettings; 377 if (!settings.serverPathPrefix.endsWith("/")) settings.serverPathPrefix ~= "/"; 378 379 void callback(scope HTTPServerRequest req, scope HTTPServerResponse res) 380 { 381 string srv_path; 382 if (auto pp = "pathMatch" in req.params) srv_path = *pp; 383 else if (req.path.length > 0) srv_path = req.path; 384 else srv_path = req.requestURL; 385 386 if (!srv_path.startsWith(settings.serverPathPrefix)) { 387 logDebug("path '%s' not starting with '%s'", srv_path, settings.serverPathPrefix); 388 return; 389 } 390 391 auto rel_path = srv_path[settings.serverPathPrefix.length .. $]; 392 auto rpath = Path(rel_path); 393 logTrace("Processing '%s'", srv_path); 394 395 rpath.normalize(); 396 logDebug("Path '%s' -> '%s'", rel_path, rpath.toNativeString()); 397 if (rpath.absolute) { 398 logDebug("Path is absolute, not responding"); 399 return; 400 } else if (!rpath.empty && rpath[0] == "..") 401 return; // don't respond to relative paths outside of the root path 402 403 sendFileCacheImpl(req, res, local_path ~ rpath, settings); 404 } 405 406 return &callback; 407 } 408 /// ditto 409 HTTPServerRequestDelegateS serveStaticFiles(string local_path, HTTPFileServerSettings settings = null) 410 { 411 return serveStaticFiles(Path(local_path), settings); 412 } 413 414 /** 415 Returns a request handler that serves a specific file on disk. 416 417 See `sendFile` for more information. 418 419 Params: 420 local_path = Path to the file to serve. 421 settings = Optional settings object enabling customization of how 422 the file gets served. 423 424 Returns: 425 A request delegate is returned, which is suitable for registering in 426 a `URLRouter` or for passing to `listenHTTP`. 427 428 See_Also: `serveStaticFiles`, `sendFile` 429 */ 430 HTTPServerRequestDelegateS serveStaticFile(Path local_path, HTTPFileServerSettings settings = null) 431 { 432 if (!settings) settings = new HTTPFileServerSettings; 433 assert(settings.serverPathPrefix == "/", "serverPathPrefix is not supported for single file serving."); 434 435 void callback(scope HTTPServerRequest req, scope HTTPServerResponse res) 436 { 437 sendFileCacheImpl(req, res, local_path, settings); 438 } 439 440 return &callback; 441 } 442 /// ditto 443 HTTPServerRequestDelegateS serveStaticFile(string local_path, HTTPFileServerSettings settings = null) 444 { 445 return serveStaticFile(Path(local_path), settings); 446 }