Add support for precompressed static files

By setting use_gzip_static to true in deflate options, in a vhost configuration, It is possible to serve precompressed versions of static files. Yaws will look for precompressed files in the same location as original files that end in ".gz". Only files that do not fit in the cache are concerned and the mtime of a precompressed file must be higher than the one of original file.
GokulEvuri · May 11, 2012 · 8981ce2 · 8981ce2
1 parent ea529f1
commit 8981ce2
Show file tree

Hide file tree

Showing 9 changed files with 157 additions and 36 deletions.
diff --git a/.gitignore b/.gitignore
@@ -31,6 +31,7 @@ test/t[12345]/yaws.conf
 www/yaws.pdf
 www/yaws.ps
 www/*.txt
+www/*.txt.*
 www/test.php
 applications/yapp/doc/edoc-info
 applications/yapp/doc/erlang.png

diff --git a/doc/yaws.tex b/doc/yaws.tex
@@ -2500,6 +2500,12 @@ \section{Server Part}
                \verb+zlib(3erl)+ for more details on the \verb+strategy+
                parameter. The default value is default.
 
+             \item \verb+use_gzip_static = true | false+ --- If true,
+               \Yaws\ will try to serve precompressed versions of static
+               files. It will look for precompressed files in the same location
+               as original files that end in ".gz". Only files that do not fit
+               in the cache are concerned. The default value is false.
+
              \item \verb+mime_types = ListOfTypes | defaults | all+ ---
                Restricts the deflate compression to particular mime types. The
                special value all enable it for all types (It is a synonym of

diff --git a/include/yaws.hrl b/include/yaws.hrl
@@ -281,6 +281,7 @@
           window_size       = -15,     % -15..-9
           mem_level         = 8,       % 1..9
           strategy          = default, % default | filtered | huffman_only
+          use_gzip_static   = false,
 
           %% [{Type, undefined|SubType}] | all
           mime_types = ?DEFAULT_COMPRESSIBLE_MIME_TYPES

diff --git a/man/yaws.conf.5 b/man/yaws.conf.5
@@ -464,6 +464,15 @@ for more details on the \fIstrategy\fR parameter. The default value is
 .RE
 .HP
 
+\fBuse_gzip_static = true | false\fR
+.RS 12
+If true, Yaws will try to serve precompressed versions of static files. It will
+look for precompressed files in the same location as original files that end in
+".gz". Only files that do not fit in the cache are concerned. The default value
+is \fIfalse\fR.
+.RE
+.HP
+
 
 \fBmime_types = ListOfTypes | defaults | all\fR
 .RS 12

diff --git a/src/yaws_config.erl b/src/yaws_config.erl
@@ -1673,6 +1673,14 @@ fload(FD, server_deflate, GC, C, Cs, Lno, Chars, Deflate) ->
                     {error,
                      ?F("Unknown strategy ~p at line ~w", [Strategy, Lno])}
             end;
+        ["use_gzip_static", '=', Bool] ->
+            case is_bool(Bool) of
+                {true, Val} ->
+                    D2 = Deflate#deflate{use_gzip_static=Val},
+                    fload(FD, server_deflate, GC, C, Cs, Lno+1, Next, D2);
+                false ->
+                    {error, ?F("Expect true|false at line ~w", [Lno])}
+            end;
         ['<', "/deflate", '>'] ->
             D2 = case Deflate#deflate.mime_types of
                      [] ->

diff --git a/src/yaws_server.erl b/src/yaws_server.erl
@@ -3528,7 +3528,7 @@ decide_deflate(true, SC, Arg, Data, decide, Mode) ->
                             yaws:outh_set_content_encoding(deflate),
                             {ok, DB} = yaws_zlib:gzip(to_binary(Data), DOpts),
                             {data, DB};
-                        true when Mode =:= stream ->
+                        true -> %% Mode == stream | {file,_,_}
                             ?Debug("Compress streamed data~n", []),
                             yaws:outh_set_content_encoding(deflate),
                             true;
@@ -3555,8 +3555,8 @@ deliver_accumulated(Sock) ->
     deliver_accumulated(undefined, Sock, undefined, final).
 
 %% Arg           = #arg{} | undefined
-%% ContentLength = Int | undefined
-%% Mode          = final | stream
+%% ContentLength = Int    | undefined
+%% Mode          = final  | stream | {file, File, MTime}
 %%
 %% For Mode==final: (all content has been accumulated before calling
 %%                   deliver_accumulated)
@@ -3565,6 +3565,11 @@ deliver_accumulated(Sock) ->
 %% For Mode==stream:
 %%     Result: opaque value to be threaded through
 %%             send_streamcontent_chunk / end_streaming
+%%
+%% For Mode=={file,File,MTime}:
+%%     Result: {gzfile, GzFile} is gzip_static option is enabled and if
+%%     GzFile exists. Else, same result than for Mode==stream
+
 deliver_accumulated(Arg, Sock, ContentLength, Mode) ->
     %% See if we must close the connection
     receive
@@ -3595,39 +3600,58 @@ deliver_accumulated(Arg, Sock, ContentLength, Mode) ->
     Result.
 
 deflate_accumulated(Arg, Content, ContentLength, Mode) ->
-    SC  = get(sc),
-    Enc = yaws:outh_get_content_encoding(),
+    SC    = get(sc),
+    Enc   = yaws:outh_get_content_encoding(),
+    DOpts = SC#sconf.deflate_options,
     {Result, Data, Size} =
         case decide_deflate(?sc_has_deflate(SC), SC, Arg, Content, Enc, Mode) of
-            {data, Bin} -> % implies Mode==final
+            {data, Bin} ->
+                %% implies Mode==final
                 {undefined, Bin, binary_size(Bin)};
-            true -> % implies Mode==stream
+
+            true when Mode == stream; DOpts#deflate.use_gzip_static == false ->
                 Z = zlib:open(),
                 {ok, Priv, Bin} =
-                    yaws_zlib:gzipDeflate(
-                      Z, yaws_zlib:gzipInit(Z, SC#sconf.deflate_options),
-                      to_binary(Content), none
-                     ),
+                    yaws_zlib:gzipDeflate(Z,yaws_zlib:gzipInit(Z,DOpts),
+                                          to_binary(Content),none),
                 {{Z, Priv}, Bin, undefined};
+            true ->
+                %% implies Mode=={file,_,_} and use_gzip_static==true
+                {file, File, MTime} = Mode,
+                GzFile = File++".gz",
+                case prim_file:read_file_info(GzFile) of
+                    {ok, FI} when FI#file_info.type == regular,
+                                  FI#file_info.mtime >= MTime ->
+                        {{gzfile, GzFile}, <<>>, FI#file_info.size};
+                    _ ->
+                        Z = zlib:open(),
+                        {ok, Priv, Bin} =
+                            yaws_zlib:gzipDeflate(Z,yaws_zlib:gzipInit(Z,DOpts),
+                                                  to_binary(Content),none),
+                        {{Z, Priv}, Bin, undefined}
+                end;
+
+            false when Mode == final ->
+                {undefined, Content, binary_size(Content)};
             false ->
-                Sz = case Mode of
-                         final  -> binary_size(Content);
-                         stream -> ContentLength
-                     end,
-                {undefined, Content, Sz}
+                %% implies Mode=stream | {file,_,_}
+                {undefined, Content, ContentLength}
         end,
     case Size of
         undefined -> yaws:outh_fix_doclose();
         _         -> yaws:accumulate_header({content_length, Size})
     end,
-    case make_chunk(Data) of
-        empty ->
-            {Result, []};
-        {S, Chunk} when Mode =:= stream ->
-            yaws:outh_inc_act_contlen(S),
-            {Result, Chunk};
+    case Mode of
+        final ->
+            {Result, Data};
         _ ->
-            {Result, Data}
+            case make_chunk(Data) of
+                empty ->
+                    {Result, []};
+                {S, Chunk} ->
+                    yaws:outh_inc_act_contlen(S),
+                    {Result, Chunk}
+            end
     end.
 
 
@@ -3782,7 +3806,8 @@ deliver_large_file(CliSock,  _Req, UT, Range) ->
                  yaws:outh_set_content_encoding(identity),
                  (To - From + 1)
          end,
-    case deliver_accumulated(undefined, CliSock, Sz, stream) of
+    Mode = {file, UT#urltype.fullpath, mtime(UT#urltype.finfo)},
+    case deliver_accumulated(undefined, CliSock, Sz, Mode) of
         discard -> ok;
         Priv    -> send_file(CliSock, UT#urltype.fullpath, Range, Priv)
     end,
@@ -3791,16 +3816,26 @@ deliver_large_file(CliSock,  _Req, UT, Range) ->
 
 send_file(CliSock, Path, all, undefined) when is_port(CliSock) ->
     ?Debug("send_file(~p,~p,no ...)~n", [CliSock, Path]),
-    yaws_sendfile:send(CliSock, Path),
-    {ok, Size} = yaws:filesize(Path),
+    {ok, Size} = yaws_sendfile:send(CliSock, Path),
+    yaws_stats:sent(Size);
+send_file(CliSock, Path, all, undefined) ->
+    ?Debug("send_file(~p,~p,no ...)~n", [CliSock, Path]),
+    {ok, Fd} = file:open(Path, [raw, binary, read]),
+    send_file(CliSock, Fd, undefined);
+send_file(CliSock, _, all, {gzfile, GzFile}) when is_port(CliSock) ->
+    ?Debug("send_file(~p,~p, ...)~n", [CliSock, GzFile]),
+    {ok, Size} = yaws_sendfile:send(CliSock, GzFile),
     yaws_stats:sent(Size);
+send_file(CliSock, _, all, {gzfile, GzFile}) ->
+    ?Debug("send_file(~p,~p, ...)~n", [CliSock, GzFile]),
+    {ok, Fd} = file:open(GzFile, [raw, binary, read]),
+    send_file(CliSock, Fd, undefined);
 send_file(CliSock, Path, all, Priv) ->
     ?Debug("send_file(~p,~p, ...)~n", [CliSock, Path]),
     {ok, Fd} = file:open(Path, [raw, binary, read]),
     send_file(CliSock, Fd, Priv);
 send_file(CliSock, Path,  {fromto, From, To, _Tot}, _) when is_port(CliSock) ->
-    Size = To - From + 1,
-    yaws_sendfile:send(CliSock, Path, From, Size),
+    {ok, Size} = yaws_sendfile:send(CliSock, Path, From, (To-From+1)),
     yaws_stats:sent(Size);
 send_file(CliSock, Path,  {fromto, From, To, _Tot}, _) ->
     {ok, Fd} = file:open(Path, [raw, binary, read]),

diff --git a/test/conf/deflate.conf b/test/conf/deflate.conf
@@ -180,3 +180,15 @@ max_size_cached_file = 5120000
                 window_size = 9
         </deflate>
 </server>
+
+
+<server localhost>
+        port = 8007
+        listen = 0.0.0.0
+        listen_backlog = 512
+        deflate = true
+        docroot = %YTOP%/www
+        <deflate>
+                use_gzip_static = true
+        </deflate>
+</server>
diff --git a/test/t5/Makefile b/test/t5/Makefile
@@ -18,6 +18,10 @@ test: all start
 	dd if=/dev/zero of=../../www/1000.txt bs=1024 count=1000 >/dev/null 2>&1
 	dd if=/dev/zero of=../../www/3000.txt bs=1024 count=3000 >/dev/null 2>&1
 	dd if=/dev/zero of=../../www/10000.txt bs=1024 count=10000 >/dev/null 2>&1
+	gzip -c ../../www/10000.txt > ../../www/10000.txt.gz
+	cp ../../www/10000.txt.gz ../../www/10000.txt.old.gz
+	sleep 1
+	gunzip -c ../../www/10000.txt.old.gz > ../../www/10000.txt.old
 	ul=`ulimit -n` ; \
 	val=`expr $$ul '<' $(ULIMIT)` ; \
 	if [ $$val = 1 ] ; then \
@@ -34,5 +38,6 @@ debug:
 	$(ERL) $(PA)
 
 clean: tclean
-	-rm -f ../../www/0.txt ../../www/1000.txt ../../www/10000.txt
+	-rm -f ../../www/0.txt ../../www/1000.txt ../../www/10000.txt ../../www/10000.txt.old
+	-rm -f ../../www/10000.txt.gz ../../www/10000.txt.old.gz
 	-rm -rf localhost:8000 logs yaws.conf
diff --git a/test/t5/app_test.erl b/test/t5/app_test.erl
@@ -129,6 +129,7 @@ deflate_options() ->
     io:format("deflate_options\n", []),
     deflate_mime_types(),
     deflate_compress_size(),
+    deflate_gzip_static(),
     deflate_otheroptions(),
     ok.
 
@@ -191,9 +192,10 @@ deflate_compress_size() ->
     ?line undefined = proplists:get_value("Content-Encoding", Hdrs1),
 
     Uri2 = "http://localhost:8005/3000.txt",
-    ?line {ok, "200", Hdrs2, _} =
+    ?line {ok, "200", Hdrs2, Body2} =
         ibrowse:send_req(Uri2, [{"Accept-Encoding", "gzip, deflate"}], get),
     ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs2),
+    ?line true = is_binary(zlib:gunzip(Body2)),
 
     %% Dynamic content
     Uri3 = "http://localhost:8005/smalltest",
@@ -202,9 +204,46 @@ deflate_compress_size() ->
     ?line undefined = proplists:get_value("Content-Encoding", Hdrs3),
 
     Uri4 = "http://localhost:8005/bigtest",
-    ?line {ok, "200", Hdrs4, _} =
+    ?line {ok, "200", Hdrs4, Body4} =
         ibrowse:send_req(Uri4, [{"Accept-Encoding", "gzip, deflate"}], get),
     ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs4),
+    ?line true = is_binary(zlib:gunzip(Body4)),
+    ok.
+
+
+deflate_gzip_static() ->
+    io:format("  deflate_gzip_static\n", []),
+
+    %% when gzip_static is disabled, large static files are chunked
+    Uri1 = "http://localhost:8006/10000.txt",
+    ?line {ok, "200", Hdrs1, Body1} =
+        ibrowse:send_req(Uri1, [{"Accept-Encoding", "gzip, deflate"}], get),
+    ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs1),
+    ?line "chunked" = proplists:get_value("Transfer-Encoding", Hdrs1),
+    ?line undefined = proplists:get_value("Content-Length", Hdrs1),
+    ?line true = is_binary(zlib:gunzip(Body1)),
+
+    %% when gzip_static is enabled, if precompressed static file is found, the
+    %% response is not chunked
+    Uri2 = "http://localhost:8007/10000.txt",
+    ?line {ok, "200", Hdrs2, Body2} =
+        ibrowse:send_req(Uri2, [{"Accept-Encoding", "gzip, deflate"}], get),
+    ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs2),
+    ?line undefined = proplists:get_value("Transfer-Encoding", Hdrs2),
+    ?line true = is_binary(zlib:gunzip(Body2)),
+
+    ?line zlib:gunzip(Body1) == zlib:gunzip(Body2),
+
+    %% if mtimes of compressed and uncompress files do not match, the compressed
+    %% file is ignored
+    Uri3 = "http://localhost:8007/10000.txt.old",
+    ?line {ok, "200", Hdrs3, Body3} =
+        ibrowse:send_req(Uri3, [{"Accept-Encoding", "gzip, deflate"}], get),
+    ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs3),
+    ?line "chunked" = proplists:get_value("Transfer-Encoding", Hdrs3),
+    ?line undefined = proplists:get_value("Content-Length", Hdrs3),
+    ?line true = is_binary(zlib:gunzip(Body3)),
+
     ok.
 
 
@@ -213,32 +252,37 @@ deflate_otheroptions() ->
 
     %% Static content
     Uri1 = "http://localhost:8006/1000.txt",
-    ?line {ok, "200", Hdrs1, _} =
+    ?line {ok, "200", Hdrs1, Body1} =
         ibrowse:send_req(Uri1, [{"Accept-Encoding", "gzip, deflate"}], get),
     ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs1),
+    ?line true = is_binary(zlib:gunzip(Body1)),
 
     Uri2 = "http://localhost:8006/10000.txt",
-    ?line {ok, "200", Hdrs2, _} =
+    ?line {ok, "200", Hdrs2, Body2} =
         ibrowse:send_req(Uri2, [{"Accept-Encoding", "gzip, deflate"}], get),
     ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs2),
     ?line "chunked" = proplists:get_value("Transfer-Encoding", Hdrs2),
     ?line undefined = proplists:get_value("Content-Length", Hdrs2),
+    ?line true = is_binary(zlib:gunzip(Body2)),
 
     %% Dynamic content
     Uri3 = "http://localhost:8006/smalltest",
-    ?line {ok, "200", Hdrs3, _} =
+    ?line {ok, "200", Hdrs3, Body3} =
         ibrowse:send_req(Uri3, [{"Accept-Encoding", "gzip, deflate"}], get),
     ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs3),
+    ?line true = is_binary(zlib:gunzip(Body3)),
 
     Uri4 = "http://localhost:8006/bigtest",
-    ?line {ok, "200", Hdrs4, _} =
+    ?line {ok, "200", Hdrs4, Body4} =
         ibrowse:send_req(Uri4, [{"Accept-Encoding", "gzip, deflate"}], get),
     ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs4),
+    ?line true = is_binary(zlib:gunzip(Body4)),
 
     Uri5 = "http://localhost:8006/streamtest",
-    ?line {ok, "200", Hdrs5, _} =
+    ?line {ok, "200", Hdrs5, Body5} =
         ibrowse:send_req(Uri5, [{"Accept-Encoding", "gzip, deflate"}], get),
     ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs5),
     ?line "chunked" = proplists:get_value("Transfer-Encoding", Hdrs5),
     ?line undefined = proplists:get_value("Content-Length", Hdrs5),
+    ?line true = is_binary(zlib:gunzip(Body5)),
     ok.