From 248ec3f83401528152c3b34dd81e994563706fea Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 23 Dec 2019 21:29:34 +0100 Subject: net.http.parser: Silence warning about unused variable [luacheck] --- net/http/parser.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'net/http/parser.lua') diff --git a/net/http/parser.lua b/net/http/parser.lua index 4e4ae9fb..62c09aef 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -63,7 +63,8 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) if buftable then buf, buftable = t_concat(buf), false; end local index = buf:find("\r\n\r\n", nil, true); if not index then return; end -- not enough data - local method, path, httpversion, status_code, reason_phrase; + -- FIXME was reason_phrase meant to be passed on somewhere? + local method, path, httpversion, status_code, reason_phrase; -- luacheck: ignore reason_phrase local first_line; local headers = {}; for line in buf:sub(1,index+1):gmatch("([^\r\n]+)\r\n") do -- parse request -- cgit v1.2.3 From 48bb417812710633ce1340e1f0f91366905b2a4f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 1 Jan 2020 01:22:57 +0100 Subject: net.http.parser: Add TODO related to #726 --- net/http/parser.lua | 1 + 1 file changed, 1 insertion(+) (limited to 'net/http/parser.lua') diff --git a/net/http/parser.lua b/net/http/parser.lua index 62c09aef..b328bdfe 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -93,6 +93,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) chunked = have_body and headers["transfer-encoding"] == "chunked"; len = tonumber(headers["content-length"]); -- TODO check for invalid len if len and len > bodylimit then error = true; return error_cb("content-length-limit-exceeded"); end + -- TODO ask a callback whether to proceed in case of large requests or Expect: 100-continue if client then -- FIXME handle '100 Continue' response (by skipping it) if not have_body then len = 0; end -- cgit v1.2.3 From 64aa6a2a0e704221821cb53ccd90e19fabfa475e Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 1 Aug 2020 18:14:09 +0200 Subject: net.http.parser: Switch to util.dbuffer for buffering incoming data This is primarily a step towards saving uploads directly to files, tho this should hopefully be more efficient than collapsing the entire buffer to a single string every now and then. --- net/http/parser.lua | 110 +++++++++++++++++++++++----------------------------- 1 file changed, 49 insertions(+), 61 deletions(-) (limited to 'net/http/parser.lua') diff --git a/net/http/parser.lua b/net/http/parser.lua index b328bdfe..84b1e005 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -1,8 +1,8 @@ local tonumber = tonumber; local assert = assert; -local t_insert, t_concat = table.insert, table.concat; local url_parse = require "socket.url".parse; local urldecode = require "util.http".urldecode; +local dbuffer = require "util.dbuffer"; local function preprocess_path(path) path = urldecode((path:gsub("//+", "/"))); @@ -28,10 +28,13 @@ local httpstream = {}; function httpstream.new(success_cb, error_cb, parser_type, options_cb) local client = true; if not parser_type or parser_type == "server" then client = false; else assert(parser_type == "client", "Invalid parser type"); end - local buf, buflen, buftable = {}, 0, true; local bodylimit = tonumber(options_cb and options_cb().body_size_limit) or 10*1024*1024; + -- https://stackoverflow.com/a/686243 + -- Indiviual headers can be up to 16k? What madness? + local headlimit = tonumber(options_cb and options_cb().head_size_limit) or 10*1024; local buflimit = tonumber(options_cb and options_cb().buffer_size_limit) or bodylimit * 2; - local chunked, chunk_size, chunk_start; + local buffer = dbuffer.new(buflimit); + local chunked; local state = nil; local packet; local len; @@ -41,33 +44,26 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) feed = function(_, data) if error then return nil, "parse has failed"; end if not data then -- EOF - if buftable then buf, buftable = t_concat(buf), false; end if state and client and not len then -- reading client body until EOF - packet.body = buf; + buffer:collapse(); + packet.body = buffer:read_chunk() or ""; success_cb(packet); - elseif buf ~= "" then -- unexpected EOF + state = nil; + elseif buffer:length() ~= 0 then -- unexpected EOF error = true; return error_cb("unexpected-eof"); end return; end - if buftable then - t_insert(buf, data); - else - buf = { buf, data }; - buftable = true; - end - buflen = buflen + #data; - if buflen > buflimit then error = true; return error_cb("max-buffer-size-exceeded"); end - while buflen > 0 do + if not buffer:write(data) then error = true; return error_cb("max-buffer-size-exceeded"); end + while buffer:length() > 0 do if state == nil then -- read request - if buftable then buf, buftable = t_concat(buf), false; end - local index = buf:find("\r\n\r\n", nil, true); + local index = buffer:sub(1, headlimit):find("\r\n\r\n", nil, true); if not index then return; end -- not enough data -- FIXME was reason_phrase meant to be passed on somewhere? local method, path, httpversion, status_code, reason_phrase; -- luacheck: ignore reason_phrase local first_line; local headers = {}; - for line in buf:sub(1,index+1):gmatch("([^\r\n]+)\r\n") do -- parse request + for line in buffer:read(index+3):gmatch("([^\r\n]+)\r\n") do -- parse request if first_line then local key, val = line:match("^([^%s:]+): *(.*)$"); if not key then error = true; return error_cb("invalid-header-line"); end -- TODO handle multi-line and invalid headers @@ -101,7 +97,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) code = status_code; httpversion = httpversion; headers = headers; - body = have_body and "" or nil; + body = false; -- COMPAT the properties below are deprecated responseversion = httpversion; responseheaders = headers; @@ -126,60 +122,52 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) path = path; httpversion = httpversion; headers = headers; - body = nil; + body = false; + body_sink = nil; }; end - buf = buf:sub(index + 4); - buflen = #buf; + if chunked then + packet.body_buffer = dbuffer.new(buflimit); + end state = true; end if state then -- read body - if client then - if chunked then - if chunk_start and buflen - chunk_start - 2 < chunk_size then - return; - end -- not enough data - if buftable then buf, buftable = t_concat(buf), false; end - if not buf:find("\r\n", nil, true) then - return; - end -- not enough data - if not chunk_size then - chunk_size, chunk_start = buf:match("^(%x+)[^\r\n]*\r\n()"); - chunk_size = chunk_size and tonumber(chunk_size, 16); - if not chunk_size then error = true; return error_cb("invalid-chunk-size"); end - end - if chunk_size == 0 and buf:find("\r\n\r\n", chunk_start-2, true) then - state, chunk_size = nil, nil; - buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped - success_cb(packet); - elseif buflen - chunk_start - 2 >= chunk_size then -- we have a chunk - packet.body = packet.body..buf:sub(chunk_start, chunk_start + (chunk_size-1)); - buf = buf:sub(chunk_start + chunk_size + 2); - buflen = buflen - (chunk_start + chunk_size + 2 - 1); - chunk_size, chunk_start = nil, nil; - else -- Partial chunk remaining - break; + if chunked then + local chunk_header = buffer:sub(1, 512); -- XXX How large do chunk headers grow? + local chunk_size, chunk_start = chunk_header:match("^(%x+)[^\r\n]*\r\n()"); + if not chunk_size then return; end + chunk_size = chunk_size and tonumber(chunk_size, 16); + if not chunk_size then error = true; return error_cb("invalid-chunk-size"); end + if chunk_size == 0 and chunk_header:find("\r\n\r\n", chunk_start-2, true) then + local body_buffer = packet.body_buffer; + if body_buffer then + packet.body_buffer = nil; + body_buffer:collapse(); + packet.body = body_buffer:read_chunk() or ""; end - elseif len and buflen >= len then - if buftable then buf, buftable = t_concat(buf), false; end - if packet.code == 101 then - packet.body, buf, buflen, buftable = buf, {}, 0, true; - else - packet.body, buf = buf:sub(1, len), buf:sub(len + 1); - buflen = #buf; - end - state = nil; success_cb(packet); - else + + buffer:collapse(); + local buf = buffer:read_chunk(); + buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped + buffer:write(buf); + state, chunked = nil, nil; + success_cb(packet); + elseif buffer:length() - chunk_start - 2 >= chunk_size then -- we have a chunk + buffer:discard(chunk_start - 1); -- TODO verify that it's not off-by-one + packet.body_buffer:write(buffer:read(chunk_size)); + buffer:discard(2); -- CRLF + else -- Partial chunk remaining break; end - elseif buflen >= len then - if buftable then buf, buftable = t_concat(buf), false; end - packet.body, buf = buf:sub(1, len), buf:sub(len + 1); - buflen = #buf; + elseif buffer:length() >= len then + assert(not chunked) + packet.body = buffer:read(len) or ""; state = nil; success_cb(packet); else break; end + else + break; end end end; -- cgit v1.2.3 From 91d2ab91086d2aebcbc4d47a5bce05c6cd3abdcb Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 1 Aug 2020 18:41:23 +0200 Subject: net.http.parser: Allow specifying sink for large request bodies This enables uses such as saving uploaded files directly to a file on disk or streaming parsing of payloads. See #726 --- net/http/parser.lua | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) (limited to 'net/http/parser.lua') diff --git a/net/http/parser.lua b/net/http/parser.lua index 84b1e005..b8396518 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -88,8 +88,6 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) if not first_line then error = true; return error_cb("invalid-status-line"); end chunked = have_body and headers["transfer-encoding"] == "chunked"; len = tonumber(headers["content-length"]); -- TODO check for invalid len - if len and len > bodylimit then error = true; return error_cb("content-length-limit-exceeded"); end - -- TODO ask a callback whether to proceed in case of large requests or Expect: 100-continue if client then -- FIXME handle '100 Continue' response (by skipping it) if not have_body then len = 0; end @@ -126,9 +124,17 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) body_sink = nil; }; end - if chunked then + if len and len > bodylimit then + -- Early notification, for redirection + success_cb(packet); + if not packet.body_sink then error = true; return error_cb("content-length-limit-exceeded"); end + end + if chunked and not packet.body_sink then + success_cb(packet); + if not packet.body_sink then packet.body_buffer = dbuffer.new(buflimit); end + end state = true; end if state then -- read body @@ -154,11 +160,23 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) success_cb(packet); elseif buffer:length() - chunk_start - 2 >= chunk_size then -- we have a chunk buffer:discard(chunk_start - 1); -- TODO verify that it's not off-by-one - packet.body_buffer:write(buffer:read(chunk_size)); + (packet.body_sink or packet.body_buffer):write(buffer:read(chunk_size)); buffer:discard(2); -- CRLF else -- Partial chunk remaining break; end + elseif packet.body_sink then + local chunk = buffer:read_chunk(len); + while chunk and len > 0 do + if packet.body_sink:write(chunk) then + len = len - #chunk; + chunk = buffer:read_chunk(len); + else + error = true; + return error_cb("body-sink-write-failure"); + end + end + if len == 0 then state = nil; success_cb(packet); end elseif buffer:length() >= len then assert(not chunked) packet.body = buffer:read(len) or ""; -- cgit v1.2.3 From 031a8d8e6458cb2c5ba115a75cd4f404326f7acf Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 20 Aug 2020 16:43:27 +0200 Subject: net.http.parser: Fix indentation Probably due to a rebase/merge with a merge tool that ignores whitespace. Happens all the time to me :( --- net/http/parser.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'net/http/parser.lua') diff --git a/net/http/parser.lua b/net/http/parser.lua index b8396518..3470ffb1 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -132,8 +132,8 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) if chunked and not packet.body_sink then success_cb(packet); if not packet.body_sink then - packet.body_buffer = dbuffer.new(buflimit); - end + packet.body_buffer = dbuffer.new(buflimit); + end end state = true; end -- cgit v1.2.3 From bc402b6409639f130f2ef7f13c9e240194de859c Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 21 Oct 2020 10:34:16 +0100 Subject: net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets --- net/http/parser.lua | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'net/http/parser.lua') diff --git a/net/http/parser.lua b/net/http/parser.lua index 3470ffb1..96f17fdb 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -47,6 +47,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) if state and client and not len then -- reading client body until EOF buffer:collapse(); packet.body = buffer:read_chunk() or ""; + packet.partial = nil; success_cb(packet); state = nil; elseif buffer:length() ~= 0 then -- unexpected EOF @@ -96,6 +97,9 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) httpversion = httpversion; headers = headers; body = false; + body_length = len; + chunked = chunked; + partial = true; -- COMPAT the properties below are deprecated responseversion = httpversion; responseheaders = headers; @@ -122,6 +126,8 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) headers = headers; body = false; body_sink = nil; + chunked = chunked; + partial = true; }; end if len and len > bodylimit then @@ -157,6 +163,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped buffer:write(buf); state, chunked = nil, nil; + packet.partial = nil; success_cb(packet); elseif buffer:length() - chunk_start - 2 >= chunk_size then -- we have a chunk buffer:discard(chunk_start - 1); -- TODO verify that it's not off-by-one @@ -176,11 +183,17 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) return error_cb("body-sink-write-failure"); end end - if len == 0 then state = nil; success_cb(packet); end + if len == 0 then + state = nil; + packet.partial = nil; + success_cb(packet); + end elseif buffer:length() >= len then assert(not chunked) packet.body = buffer:read(len) or ""; - state = nil; success_cb(packet); + state = nil; + packet.partial = nil; + success_cb(packet); else break; end -- cgit v1.2.3