From 47ed467f4f7dbab0615054413e18d32846c91740 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Sat, 13 Aug 2016 20:19:08 +0200
Subject: net.http.parser: Buffer into a table to reduce GC pressure, collapse
to string when needed (fixes #603)
---
net/http/parser.lua | 29 ++++++++++++++++++++++-------
1 file changed, 22 insertions(+), 7 deletions(-)
diff --git a/net/http/parser.lua b/net/http/parser.lua
index 6d7187da..af43e7a0 100644
--- a/net/http/parser.lua
+++ b/net/http/parser.lua
@@ -1,5 +1,6 @@
local tonumber = tonumber;
local assert = assert;
+local t_insert, t_concat = table.insert, table.concat;
local url_parse = require "socket.url".parse;
local urldecode = require "util.http".urldecode;
@@ -27,7 +28,7 @@ local httpstream = {};
function httpstream.new(success_cb, error_cb, parser_type, options_cb)
local client = true;
if not parser_type or parser_type == "server" then client = false; else assert(parser_type == "client", "Invalid parser type"); end
- local buf = "";
+ local buf, buflen, buftable = {}, 0, true;
local chunked, chunk_size, chunk_start;
local state = nil;
local packet;
@@ -38,6 +39,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
feed = function(self, data)
if error then return nil, "parse has failed"; end
if not data then -- EOF
+ if buftable then buf, buftable = t_concat(buf), false; end
if state and client and not len then -- reading client body until EOF
packet.body = buf;
success_cb(packet);
@@ -46,9 +48,16 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
end
return;
end
- buf = buf..data;
- while #buf > 0 do
+ if buftable then
+ t_insert(buf, data);
+ else
+ buf = { buf, data };
+ buftable = true;
+ end
+ buflen = buflen + #data;
+ while buflen > 0 do
if state == nil then -- read request
+ if buftable then buf, buftable = t_concat(buf), false; end
local index = buf:find("\r\n\r\n", nil, true);
if not index then return; end -- not enough data
local method, path, httpversion, status_code, reason_phrase;
@@ -115,11 +124,13 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
};
end
buf = buf:sub(index + 4);
+ buflen = #buf;
state = true;
end
if state then -- read body
if client then
if chunked then
+ if buftable then buf, buftable = t_concat(buf), false; end
if not buf:find("\r\n", nil, true) then
return;
end -- not enough data
@@ -132,25 +143,29 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
state, chunk_size = nil, nil;
buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped
success_cb(packet);
- elseif #buf - chunk_start - 2 >= chunk_size then -- we have a chunk
+ elseif buflen - chunk_start - 2 >= chunk_size then -- we have a chunk
packet.body = packet.body..buf:sub(chunk_start, chunk_start + (chunk_size-1));
buf = buf:sub(chunk_start + chunk_size + 2);
chunk_size, chunk_start = nil, nil;
else -- Partial chunk remaining
break;
end
- elseif len and #buf >= len then
+ elseif len and buflen >= len then
+ if buftable then buf, buftable = t_concat(buf), false; end
if packet.code == 101 then
- packet.body, buf = buf, "";
+ packet.body, buf, buflen, buftable = buf, {}, 0, true;
else
packet.body, buf = buf:sub(1, len), buf:sub(len + 1);
+ buflen = #buf;
end
state = nil; success_cb(packet);
else
break;
end
- elseif #buf >= len then
+ elseif buflen >= len then
+ if buftable then buf, buftable = t_concat(buf), false; end
packet.body, buf = buf:sub(1, len), buf:sub(len + 1);
+ buflen = #buf;
state = nil; success_cb(packet);
else
break;
--
cgit v1.2.3
From b50763dcf67719fa26a5cd1bcfdde4e416363e74 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 18 Aug 2016 14:47:58 +0200
Subject: net.http.parser: Add a limit on content length, default to 10M
---
net/http/parser.lua | 2 ++
1 file changed, 2 insertions(+)
diff --git a/net/http/parser.lua b/net/http/parser.lua
index af43e7a0..0f764d12 100644
--- a/net/http/parser.lua
+++ b/net/http/parser.lua
@@ -29,6 +29,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
local client = true;
if not parser_type or parser_type == "server" then client = false; else assert(parser_type == "client", "Invalid parser type"); end
local buf, buflen, buftable = {}, 0, true;
+ local bodylimit = 10*1024*1024;
local chunked, chunk_size, chunk_start;
local state = nil;
local packet;
@@ -88,6 +89,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
if not first_line then error = true; return error_cb("invalid-status-line"); end
chunked = have_body and headers["transfer-encoding"] == "chunked";
len = tonumber(headers["content-length"]); -- TODO check for invalid len
+ if len and len > bodylimit then error = true; return error_cb("content-length-limit-exceeded"); end
if client then
-- FIXME handle '100 Continue' response (by skipping it)
if not have_body then len = 0; end
--
cgit v1.2.3
From bdae29d75431867aaf584ca88787233682426a9a Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 18 Aug 2016 14:48:42 +0200
Subject: net.http.parser: Add a limit on maximum buffer size, default to 20M
---
net/http/parser.lua | 2 ++
1 file changed, 2 insertions(+)
diff --git a/net/http/parser.lua b/net/http/parser.lua
index 0f764d12..e3a2554f 100644
--- a/net/http/parser.lua
+++ b/net/http/parser.lua
@@ -30,6 +30,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
if not parser_type or parser_type == "server" then client = false; else assert(parser_type == "client", "Invalid parser type"); end
local buf, buflen, buftable = {}, 0, true;
local bodylimit = 10*1024*1024;
+ local buflimit = bodylimit * 2;
local chunked, chunk_size, chunk_start;
local state = nil;
local packet;
@@ -56,6 +57,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
buftable = true;
end
buflen = buflen + #data;
+ if buflen > buflimit then error = true; return error_cb("max-buffer-size-exceeded"); end
while buflen > 0 do
if state == nil then -- read request
if buftable then buf, buftable = t_concat(buf), false; end
--
cgit v1.2.3
From 4fe3ec81e1ca2f13fd945e9afcaf5af2733ade73 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 18 Aug 2016 14:50:06 +0200
Subject: net.http.parser: Allow limits to be configurable via options callback
---
net/http/parser.lua | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/net/http/parser.lua b/net/http/parser.lua
index e3a2554f..1e698728 100644
--- a/net/http/parser.lua
+++ b/net/http/parser.lua
@@ -29,8 +29,8 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb)
local client = true;
if not parser_type or parser_type == "server" then client = false; else assert(parser_type == "client", "Invalid parser type"); end
local buf, buflen, buftable = {}, 0, true;
- local bodylimit = 10*1024*1024;
- local buflimit = bodylimit * 2;
+ local bodylimit = tonumber(options_cb and options_cb().body_size_limit) or 10*1024*1024;
+ local buflimit = tonumber(options_cb and options_cb().buffer_size_limit) or bodylimit * 2;
local chunked, chunk_size, chunk_start;
local state = nil;
local packet;
--
cgit v1.2.3
From 47fe58f5ca703f0c8f6378a7f8a46d7565bcef90 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 18 Aug 2016 14:50:39 +0200
Subject: net.http.server: Expose way to set http server options
---
net/http/server.lua | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/net/http/server.lua b/net/http/server.lua
index f091595c..32cda8aa 100644
--- a/net/http/server.lua
+++ b/net/http/server.lua
@@ -19,6 +19,7 @@ local sessions = {};
local listener = {};
local hosts = {};
local default_host;
+local options = {};
local function is_wildcard_event(event)
return event:sub(-2, -1) == "/*";
@@ -130,7 +131,10 @@ function listener.onconnect(conn)
sessions[conn] = nil;
conn:close();
end
- sessions[conn] = parser_new(success_cb, error_cb);
+ local function options_cb()
+ return options;
+ end
+ sessions[conn] = parser_new(success_cb, error_cb, "server", options_cb);
end
function listener.ondisconnect(conn)
@@ -300,6 +304,9 @@ end
function _M.fire_event(event, ...)
return events.fire_event(event, ...);
end
+function _M.set_option(name, value)
+ options[name] = value;
+end
_M.listener = listener;
_M.codes = codes;
--
cgit v1.2.3
From 1686ef5d53d3d84d35ebebab89b8d1b22dcf021e Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 18 Aug 2016 14:51:11 +0200
Subject: mod_http: Allow configuring http parser size limits
---
plugins/mod_http.lua | 3 +++
1 file changed, 3 insertions(+)
diff --git a/plugins/mod_http.lua b/plugins/mod_http.lua
index 9b574bc8..03b23480 100644
--- a/plugins/mod_http.lua
+++ b/plugins/mod_http.lua
@@ -18,6 +18,9 @@ local server = require "net.http.server";
server.set_default_host(module:get_option_string("http_default_host"));
+server.set_option("body_size_limit", module:get_option_number("http_max_content_size"));
+server.set_option("buffer_size_limit", module:get_option_number("http_max_buffer_size"));
+
local function normalize_path(path)
if path:sub(-1,-1) == "/" then path = path:sub(1, -2); end
if path:sub(1,1) ~= "/" then path = "/"..path; end
--
cgit v1.2.3