From 8994b3afd1cd62a0caf715093095b7163fdf710c Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Thu, 11 Apr 2013 17:32:59 +0100 Subject: net.http, util.http: Move definitions of urlencode/decode and formencode/decode to util.http (possible to use them without unnecessary network-related dependencies) --- net/http.lua | 48 +++++++----------------------------------------- util/http.lua | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 41 deletions(-) diff --git a/net/http.lua b/net/http.lua index ec55af92..516afe58 100644 --- a/net/http.lua +++ b/net/http.lua @@ -10,6 +10,7 @@ local socket = require "socket" local b64 = require "util.encodings".base64.encode; local url = require "socket.url" local httpstream_new = require "util.httpstream".new; +local util_http = require "util.http"; local ssl_available = pcall(require, "ssl"); @@ -70,46 +71,7 @@ function listener.ondisconnect(conn, err) requests[conn] = nil; end -function urlencode(s) return s and (s:gsub("[^a-zA-Z0-9.~_-]", function (c) return format("%%%02x", c:byte()); end)); end -function urldecode(s) return s and (s:gsub("%%(%x%x)", function (c) return char(tonumber(c,16)); end)); end - -local function _formencodepart(s) - return s and (s:gsub("%W", function (c) - if c ~= " " then - return format("%%%02x", c:byte()); - else - return "+"; - end - end)); -end - -function formencode(form) - local result = {}; - if form[1] then -- Array of ordered { name, value } - for _, field in ipairs(form) do - t_insert(result, _formencodepart(field.name).."=".._formencodepart(field.value)); - end - else -- Unordered map of name -> value - for name, value in pairs(form) do - t_insert(result, _formencodepart(name).."=".._formencodepart(value)); - end - end - return t_concat(result, "&"); -end - -function formdecode(s) - if not s:match("=") then return urldecode(s); end - local r = {}; - for k, v in s:gmatch("([^=&]*)=([^&]*)") do - k, v = k:gsub("%+", "%%20"), v:gsub("%+", "%%20"); - k, v = urldecode(k), urldecode(v); - t_insert(r, { name = k, value = v }); - r[k] = v; - end - return r; -end - -local function request_reader(request, data, startpos) +local function request_reader(request, data) if not request.parser then if not data then return; end local function success_cb(r) @@ -216,6 +178,10 @@ function destroy_request(request) end end -_M.urlencode = urlencode; +local urlencode, urldecode = util_http.urlencode, util_http.urldecode; +local formencode, formdecode = util_http.formencode, util_http.formdecode; + +_M.urlencode, _M.urldecode = urlencode, urldecode; +_M.formencode, _M.formdecode = formencode, formdecode; return _M; diff --git a/util/http.lua b/util/http.lua index 5b49d1d0..5dd636d9 100644 --- a/util/http.lua +++ b/util/http.lua @@ -7,9 +7,54 @@ local http = {}; +function http.urlencode(s) + return s and (s:gsub("[^a-zA-Z0-9.~_-]", function (c) return format("%%%02x", c:byte()); end)); +end +function http.urldecode(s) + return s and (s:gsub("%%(%x%x)", function (c) return char(tonumber(c,16)); end)); +end + +local function _formencodepart(s) + return s and (s:gsub("%W", function (c) + if c ~= " " then + return format("%%%02x", c:byte()); + else + return "+"; + end + end)); +end + +function http.formencode(form) + local result = {}; + if form[1] then -- Array of ordered { name, value } + for _, field in ipairs(form) do + t_insert(result, _formencodepart(field.name).."=".._formencodepart(field.value)); + end + else -- Unordered map of name -> value + for name, value in pairs(form) do + t_insert(result, _formencodepart(name).."=".._formencodepart(value)); + end + end + return t_concat(result, "&"); +end + +function http.formdecode(s) + if not s:match("=") then return urldecode(s); end + local r = {}; + for k, v in s:gmatch("([^=&]*)=([^&]*)") do + k, v = k:gsub("%+", "%%20"), v:gsub("%+", "%%20"); + k, v = urldecode(k), urldecode(v); + t_insert(r, { name = k, value = v }); + r[k] = v; + end + return r; +end + function http.contains_token(field, token) field = ","..field:gsub("[ \t]", ""):lower()..","; return field:find(","..token:lower()..",", 1, true) ~= nil; end + + return http; -- cgit v1.2.3 From 18889eae1b802daa52c282d5ebd1a2dfe078b250 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Thu, 11 Apr 2013 17:35:39 +0100 Subject: sessionmanager, s2smanager: Remove unused imports --- core/s2smanager.lua | 4 ++-- core/sessionmanager.lua | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/core/s2smanager.lua b/core/s2smanager.lua index e4de498a..06d3f2c9 100644 --- a/core/s2smanager.lua +++ b/core/s2smanager.lua @@ -9,8 +9,8 @@ local hosts = prosody.hosts; -local tostring, pairs, getmetatable, newproxy, setmetatable - = tostring, pairs, getmetatable, newproxy, setmetatable; +local tostring, pairs, setmetatable + = tostring, pairs, setmetatable; local logger_init = require "util.logger".init; diff --git a/core/sessionmanager.lua b/core/sessionmanager.lua index d178fb2d..98ead07f 100644 --- a/core/sessionmanager.lua +++ b/core/sessionmanager.lua @@ -24,9 +24,6 @@ local uuid_generate = require "util.uuid".generate; local initialize_filters = require "util.filters".initialize; local gettime = require "socket".gettime; -local newproxy = newproxy; -local getmetatable = getmetatable; - module "sessionmanager" function new_session(conn) -- cgit v1.2.3 From 2f3195166874613351f3376fb0d383479e8ce041 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Thu, 11 Apr 2013 17:37:37 +0100 Subject: net.http.parser: Depend on util.http instead of net.http for urlencode --- net/http/parser.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/http/parser.lua b/net/http/parser.lua index 2545b5ac..45a8b168 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -2,7 +2,7 @@ local tonumber = tonumber; local assert = assert; local url_parse = require "socket.url".parse; -local urldecode = require "net.http".urldecode; +local urldecode = require "util.http".urldecode; local function preprocess_path(path) path = urldecode((path:gsub("//+", "/"))); -- cgit v1.2.3 From 70a407116ffcc523fcd8cb5285e4f0977a1c6d35 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Thu, 11 Apr 2013 17:39:10 +0100 Subject: net.http.parser: Break when no more usable data in buffer (client part of e5ec60dfb202) --- net/http/parser.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/net/http/parser.lua b/net/http/parser.lua index 45a8b168..73a8fb6a 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -136,6 +136,8 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) elseif len and #buf >= len then packet.body, buf = buf:sub(1, len), buf:sub(len + 1); state = nil; success_cb(packet); + else + break; end elseif #buf >= len then packet.body, buf = buf:sub(1, len), buf:sub(len + 1); -- cgit v1.2.3 From 1ebc90c25bfb3e02c4b2f0bbee59c84bcc330299 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Thu, 11 Apr 2013 19:58:53 +0100 Subject: net.http.parser: Convert status_code to a number before trying to compare it to numbers --- net/http/parser.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/net/http/parser.lua b/net/http/parser.lua index 73a8fb6a..684d62fe 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -65,6 +65,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) first_line = line; if client then httpversion, status_code, reason_phrase = line:match("^HTTP/(1%.[01]) (%d%d%d) (.*)$"); + status_code = tonumber(status_code); if not status_code then error = true; return error_cb("invalid-status-line"); end have_body = not ( (options_cb and options_cb().method == "HEAD") -- cgit v1.2.3 From fec86f72cfdd76162667397ac7c82e950c1e5be7 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Thu, 11 Apr 2013 20:01:03 +0100 Subject: net.http.parser: Fix chunked encoding response parsing, and make it more robust --- net/http/parser.lua | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/net/http/parser.lua b/net/http/parser.lua index 684d62fe..34742d2b 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -1,4 +1,3 @@ - local tonumber = tonumber; local assert = assert; local url_parse = require "socket.url".parse; @@ -29,7 +28,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) local client = true; if not parser_type or parser_type == "server" then client = false; else assert(parser_type == "client", "Invalid parser type"); end local buf = ""; - local chunked; + local chunked, chunk_size, chunk_start; local state = nil; local packet; local len; @@ -71,7 +70,6 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) ( (options_cb and options_cb().method == "HEAD") or (status_code == 204 or status_code == 304 or status_code == 301) or (status_code >= 100 and status_code < 200) ); - chunked = have_body and headers["transfer-encoding"] == "chunked"; else method, path, httpversion = line:match("^(%w+) (%S+) HTTP/(1%.[01])$"); if not method then error = true; return error_cb("invalid-status-line"); end @@ -79,6 +77,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) end end if not first_line then error = true; return error_cb("invalid-status-line"); end + chunked = have_body and headers["transfer-encoding"] == "chunked"; len = tonumber(headers["content-length"]); -- TODO check for invalid len if client then -- FIXME handle '100 Continue' response (by skipping it) @@ -121,19 +120,25 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) if state then -- read body if client then if chunked then - local index = buf:find("\r\n", nil, true); - if not index then return; end -- not enough data - local chunk_size = buf:match("^%x+"); - if not chunk_size then error = true; return error_cb("invalid-chunk-size"); end - chunk_size = tonumber(chunk_size, 16); - index = index + 2; - if chunk_size == 0 then - state = nil; success_cb(packet); - elseif #buf - index + 1 >= chunk_size then -- we have a chunk - packet.body = packet.body..buf:sub(index, index + chunk_size - 1); - buf = buf:sub(index + chunk_size); + if not buf:find("\r\n", nil, true) then + return; + end -- not enough data + if not chunk_size then + chunk_size, chunk_start = buf:match("^(%x+)[^\r\n]*\r\n()"); + chunk_size = chunk_size and tonumber(chunk_size, 16); + if not chunk_size then error = true; return error_cb("invalid-chunk-size"); end + end + if chunk_size == 0 and buf:find("\r\n\r\n", chunk_start-2, true) then + state, chunk_size = nil, nil; + buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped + success_cb(packet); + elseif #buf - chunk_start + 2 >= chunk_size then -- we have a chunk + packet.body = packet.body..buf:sub(chunk_start, chunk_start + chunk_size); + buf = buf:sub(chunk_start + chunk_size + 2); + chunk_size, chunk_start = nil, nil; + else -- Partial chunk remaining + break; end - error("trailers"); -- FIXME MUST read trailers elseif len and #buf >= len then packet.body, buf = buf:sub(1, len), buf:sub(len + 1); state = nil; success_cb(packet); -- cgit v1.2.3 From 218b8844b4c558d18289e43dc2e1de7b004ca945 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Thu, 11 Apr 2013 20:24:37 +0100 Subject: net.http: Switch from util.httpstream to net.http.parser, introduces small but backwards-incompatible API changes - see http://prosody.im/doc/developers/http --- net/http.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/http.lua b/net/http.lua index 516afe58..b3bd5a67 100644 --- a/net/http.lua +++ b/net/http.lua @@ -9,7 +9,7 @@ local socket = require "socket" local b64 = require "util.encodings".base64.encode; local url = require "socket.url" -local httpstream_new = require "util.httpstream".new; +local httpstream_new = require "net.http.parser".new; local util_http = require "util.http"; local ssl_available = pcall(require, "ssl"); -- cgit v1.2.3 From dbc9b792220aac53dc5d441a8e4a1f1a77e1635b Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Thu, 11 Apr 2013 21:55:20 +0100 Subject: util.httpstream: Unused, remove --- util/httpstream.lua | 134 ---------------------------------------------------- 1 file changed, 134 deletions(-) delete mode 100644 util/httpstream.lua diff --git a/util/httpstream.lua b/util/httpstream.lua deleted file mode 100644 index 190b3ed6..00000000 --- a/util/httpstream.lua +++ /dev/null @@ -1,134 +0,0 @@ - -local coroutine = coroutine; -local tonumber = tonumber; - -local deadroutine = coroutine.create(function() end); -coroutine.resume(deadroutine); - -module("httpstream") - -local function parser(success_cb, parser_type, options_cb) - local data = coroutine.yield(); - local function readline() - local pos = data:find("\r\n", nil, true); - while not pos do - data = data..coroutine.yield(); - pos = data:find("\r\n", nil, true); - end - local r = data:sub(1, pos-1); - data = data:sub(pos+2); - return r; - end - local function readlength(n) - while #data < n do - data = data..coroutine.yield(); - end - local r = data:sub(1, n); - data = data:sub(n + 1); - return r; - end - local function readheaders() - local headers = {}; -- read headers - while true do - local line = readline(); - if line == "" then break; end -- headers done - local key, val = line:match("^([^%s:]+): *(.*)$"); - if not key then coroutine.yield("invalid-header-line"); end -- TODO handle multi-line and invalid headers - key = key:lower(); - headers[key] = headers[key] and headers[key]..","..val or val; - end - return headers; - end - - if not parser_type or parser_type == "server" then - while true do - -- read status line - local status_line = readline(); - local method, path, httpversion = status_line:match("^(%S+)%s+(%S+)%s+HTTP/(%S+)$"); - if not method then coroutine.yield("invalid-status-line"); end - path = path:gsub("^//+", "/"); -- TODO parse url more - local headers = readheaders(); - - -- read body - local len = tonumber(headers["content-length"]); - len = len or 0; -- TODO check for invalid len - local body = readlength(len); - - success_cb({ - method = method; - path = path; - httpversion = httpversion; - headers = headers; - body = body; - }); - end - elseif parser_type == "client" then - while true do - -- read status line - local status_line = readline(); - local httpversion, status_code, reason_phrase = status_line:match("^HTTP/(%S+)%s+(%d%d%d)%s+(.*)$"); - status_code = tonumber(status_code); - if not status_code then coroutine.yield("invalid-status-line"); end - local headers = readheaders(); - - -- read body - local have_body = not - ( (options_cb and options_cb().method == "HEAD") - or (status_code == 204 or status_code == 304 or status_code == 301) - or (status_code >= 100 and status_code < 200) ); - - local body; - if have_body then - local len = tonumber(headers["content-length"]); - if headers["transfer-encoding"] == "chunked" then - body = ""; - while true do - local chunk_size = readline():match("^%x+"); - if not chunk_size then coroutine.yield("invalid-chunk-size"); end - chunk_size = tonumber(chunk_size, 16) - if chunk_size == 0 then break; end - body = body..readlength(chunk_size); - if readline() ~= "" then coroutine.yield("invalid-chunk-ending"); end - end - local trailers = readheaders(); - elseif len then -- TODO check for invalid len - body = readlength(len); - else -- read to end - repeat - local newdata = coroutine.yield(); - data = data..newdata; - until newdata == ""; - body, data = data, ""; - end - end - - success_cb({ - code = status_code; - httpversion = httpversion; - headers = headers; - body = body; - }); - end - else coroutine.yield("unknown-parser-type"); end -end - -function new(success_cb, error_cb, parser_type, options_cb) - local co = coroutine.create(parser); - coroutine.resume(co, success_cb, parser_type, options_cb) - return { - feed = function(self, data) - if not data then - if parser_type == "client" then coroutine.resume(co, ""); end - co = deadroutine; - return error_cb(); - end - local success, result = coroutine.resume(co, data); - if result then - co = deadroutine; - return error_cb(result); - end - end; - }; -end - -return _M; -- cgit v1.2.3 From ef9b86202eb5ed42cefe788b01b1f09895d6899e Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Fri, 12 Apr 2013 00:31:05 +0100 Subject: net.http: Swap response and request parameters passed to callback (will break some modules) --- net/http.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/net/http.lua b/net/http.lua index b3bd5a67..4eb4a2ac 100644 --- a/net/http.lua +++ b/net/http.lua @@ -76,8 +76,7 @@ local function request_reader(request, data) if not data then return; end local function success_cb(r) if request.callback then - for k,v in pairs(r) do request[k] = v; end - request.callback(r.body, r.code, request, r); + request.callback(r.body, r.code, r, request); request.callback = nil; end destroy_request(request); -- cgit v1.2.3