diff options
author | Matthew Wild <mwild1@gmail.com> | 2020-09-30 09:50:33 +0100 |
---|---|---|
committer | Matthew Wild <mwild1@gmail.com> | 2020-09-30 09:50:33 +0100 |
commit | 785c20f6ee7e61a5a91a8b6259623bc2a2bbffaa (patch) | |
tree | 3a37db7b805b1555d08cc61c7938cf512ff35ce0 /net/http | |
parent | e55d037cdf89efac99c6144b381c9fa880f7fb93 (diff) | |
parent | b80ff2ae4f86aa26e055890a8284b55170ef2056 (diff) | |
download | prosody-785c20f6ee7e61a5a91a8b6259623bc2a2bbffaa.tar.gz prosody-785c20f6ee7e61a5a91a8b6259623bc2a2bbffaa.zip |
Merge 0.11->trunk
Diffstat (limited to 'net/http')
-rw-r--r-- | net/http/codes.lua | 2 | ||||
-rw-r--r-- | net/http/errors.lua | 115 | ||||
-rw-r--r-- | net/http/files.lua | 149 | ||||
-rw-r--r-- | net/http/parser.lua | 132 | ||||
-rw-r--r-- | net/http/server.lua | 105 |
5 files changed, 409 insertions, 94 deletions
diff --git a/net/http/codes.lua b/net/http/codes.lua index 8098b5c3..4327f151 100644 --- a/net/http/codes.lua +++ b/net/http/codes.lua @@ -82,5 +82,5 @@ local response_codes = { -- [512-599] = "Unassigned"; }; -for k,v in pairs(response_codes) do response_codes[k] = k.." "..v; end +for k,v in pairs(response_codes) do response_codes[k] = ("%03d %s"):format(k, v); end return setmetatable(response_codes, { __index = function(_, k) return k.." Unassigned"; end }) diff --git a/net/http/errors.lua b/net/http/errors.lua new file mode 100644 index 00000000..ae45d5a8 --- /dev/null +++ b/net/http/errors.lua @@ -0,0 +1,115 @@ +-- This module returns a table that is suitable for use as a util.error registry, +-- and a function to return a util.error object given callback 'code' and 'body' +-- parameters. + +local codes = require "net.http.codes"; +local util_error = require "util.error"; + +local error_templates = { + -- This code is used by us to report a client-side or connection error. + -- Instead of using the code, use the supplied body text to get one of + -- the more detailed errors below. + [0] = { + code = 0, type = "cancel", condition = "internal-server-error"; + text = "Connection or internal error"; + }; + + -- These are net.http built-in errors, they are returned in + -- the body parameter when code == 0 + ["cancelled"] = { + code = 0, type = "cancel", condition = "remote-server-timeout"; + text = "Request cancelled"; + }; + ["connection-closed"] = { + code = 0, type = "wait", condition = "remote-server-timeout"; + text = "Connection closed"; + }; + ["certificate-chain-invalid"] = { + code = 0, type = "cancel", condition = "remote-server-timeout"; + text = "Server certificate not trusted"; + }; + ["certificate-verify-failed"] = { + code = 0, type = "cancel", condition = "remote-server-timeout"; + text = "Server certificate invalid"; + }; + ["connection failed"] = { + code = 0, type = "cancel", condition = "remote-server-not-found"; + text = "Connection failed"; + }; + ["invalid-url"] = { + code = 0, type = "modify", condition = "bad-request"; + text = "Invalid URL"; + }; + + -- This doesn't attempt to map every single HTTP code (not all have sane mappings), + -- but all the common ones should be covered. XEP-0086 was used as reference for + -- most of these. + [400] = { type = "modify", condition = "bad-request" }; + [401] = { type = "auth", condition = "not-authorized" }; + [402] = { type = "auth", condition = "payment-required" }; + [403] = { type = "auth", condition = "forbidden" }; + [404] = { type = "cancel", condition = "item-not-found" }; + [405] = { type = "cancel", condition = "not-allowed" }; + [406] = { type = "modify", condition = "not-acceptable" }; + [407] = { type = "auth", condition = "registration-required" }; + [408] = { type = "wait", condition = "remote-server-timeout" }; + [409] = { type = "cancel", condition = "conflict" }; + [410] = { type = "cancel", condition = "gone" }; + [411] = { type = "modify", condition = "bad-request" }; + [412] = { type = "cancel", condition = "conflict" }; + [413] = { type = "modify", condition = "resource-constraint" }; + [414] = { type = "modify", condition = "resource-constraint" }; + [415] = { type = "cancel", condition = "feature-not-implemented" }; + [416] = { type = "modify", condition = "bad-request" }; + + [422] = { type = "modify", condition = "bad-request" }; + [423] = { type = "wait", condition = "resource-constraint" }; + + [429] = { type = "wait", condition = "resource-constraint" }; + [431] = { type = "modify", condition = "resource-constraint" }; + [451] = { type = "auth", condition = "forbidden" }; + + [500] = { type = "wait", condition = "internal-server-error" }; + [501] = { type = "cancel", condition = "feature-not-implemented" }; + [502] = { type = "wait", condition = "remote-server-timeout" }; + [503] = { type = "cancel", condition = "service-unavailable" }; + [504] = { type = "wait", condition = "remote-server-timeout" }; + [507] = { type = "wait", condition = "resource-constraint" }; + [511] = { type = "auth", condition = "not-authorized" }; +}; + +for k, v in pairs(codes) do + if error_templates[k] then + error_templates[k].code = k; + error_templates[k].text = v; + else + error_templates[k] = { type = "cancel", condition = "undefined-condition", text = v, code = k }; + end +end + +setmetatable(error_templates, { + __index = function(_, k) + if type(k) ~= "number" then + return nil; + end + return { + type = "cancel"; + condition = "undefined-condition"; + text = codes[k] or (k.." Unassigned"); + code = k; + }; + end +}); + +local function new(code, body, context) + if code == 0 then + return util_error.new(body, context, error_templates); + else + return util_error.new(code, context, error_templates); + end +end + +return { + registry = error_templates; + new = new; +}; diff --git a/net/http/files.lua b/net/http/files.lua new file mode 100644 index 00000000..583f7514 --- /dev/null +++ b/net/http/files.lua @@ -0,0 +1,149 @@ +-- Prosody IM +-- Copyright (C) 2008-2010 Matthew Wild +-- Copyright (C) 2008-2010 Waqas Hussain +-- +-- This project is MIT/X11 licensed. Please see the +-- COPYING file in the source package for more information. +-- + +local server = require"net.http.server"; +local lfs = require "lfs"; +local new_cache = require "util.cache".new; +local log = require "util.logger".init("net.http.files"); + +local os_date = os.date; +local open = io.open; +local stat = lfs.attributes; +local build_path = require"socket.url".build_path; +local path_sep = package.config:sub(1,1); + + +local forbidden_chars_pattern = "[/%z]"; +if package.config:sub(1,1) == "\\" then + forbidden_chars_pattern = "[/%z\001-\031\127\"*:<>?|]" +end + +local urldecode = require "util.http".urldecode; +local function sanitize_path(path) --> util.paths or util.http? + if not path then return end + local out = {}; + + local c = 0; + for component in path:gmatch("([^/]+)") do + component = urldecode(component); + if component:find(forbidden_chars_pattern) then + return nil; + elseif component == ".." then + if c <= 0 then + return nil; + end + out[c] = nil; + c = c - 1; + elseif component ~= "." then + c = c + 1; + out[c] = component; + end + end + if path:sub(-1,-1) == "/" then + out[c+1] = ""; + end + return "/"..table.concat(out, "/"); +end + +local function serve(opts) + if type(opts) ~= "table" then -- assume path string + opts = { path = opts }; + end + local mime_map = opts.mime_map or { html = "text/html" }; + local cache = new_cache(opts.cache_size or 256); + local cache_max_file_size = tonumber(opts.cache_max_file_size) or 1024 + -- luacheck: ignore 431 + local base_path = opts.path; + local dir_indices = opts.index_files or { "index.html", "index.htm" }; + local directory_index = opts.directory_index; + local function serve_file(event, path) + local request, response = event.request, event.response; + local sanitized_path = sanitize_path(path); + if path and not sanitized_path then + return 400; + end + path = sanitized_path; + local orig_path = sanitize_path(request.path); + local full_path = base_path .. (path or ""):gsub("/", path_sep); + local attr = stat(full_path:match("^.*[^\\/]")); -- Strip trailing path separator because Windows + if not attr then + return 404; + end + + local request_headers, response_headers = request.headers, response.headers; + + local last_modified = os_date('!%a, %d %b %Y %H:%M:%S GMT', attr.modification); + response_headers.last_modified = last_modified; + + local etag = ('"%x-%x-%x"'):format(attr.change or 0, attr.size or 0, attr.modification or 0); + response_headers.etag = etag; + + local if_none_match = request_headers.if_none_match + local if_modified_since = request_headers.if_modified_since; + if etag == if_none_match + or (not if_none_match and last_modified == if_modified_since) then + return 304; + end + + local data = cache:get(orig_path); + if data and data.etag == etag then + response_headers.content_type = data.content_type; + data = data.data; + cache:set(orig_path, data); + elseif attr.mode == "directory" and path then + if full_path:sub(-1) ~= "/" then + local dir_path = { is_absolute = true, is_directory = true }; + for dir in orig_path:gmatch("[^/]+") do dir_path[#dir_path+1]=dir; end + response_headers.location = build_path(dir_path); + return 301; + end + for i=1,#dir_indices do + if stat(full_path..dir_indices[i], "mode") == "file" then + return serve_file(event, path..dir_indices[i]); + end + end + + if directory_index then + data = server._events.fire_event("directory-index", { path = request.path, full_path = full_path }); + end + if not data then + return 403; + end + cache:set(orig_path, { data = data, content_type = mime_map.html; etag = etag; }); + response_headers.content_type = mime_map.html; + + else + local f, err = open(full_path, "rb"); + if not f then + log("debug", "Could not open %s. Error was %s", full_path, err); + return 403; + end + local ext = full_path:match("%.([^./]+)$"); + local content_type = ext and mime_map[ext]; + response_headers.content_type = content_type; + if attr.size > cache_max_file_size then + response_headers.content_length = ("%d"):format(attr.size); + log("debug", "%d > cache_max_file_size", attr.size); + return response:send_file(f); + else + data = f:read("*a"); + f:close(); + end + cache:set(orig_path, { data = data; content_type = content_type; etag = etag }); + end + + return response:send(data); + end + + return serve_file; +end + +return { + serve = serve; +} + diff --git a/net/http/parser.lua b/net/http/parser.lua index 4e4ae9fb..3470ffb1 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -1,8 +1,8 @@ local tonumber = tonumber; local assert = assert; -local t_insert, t_concat = table.insert, table.concat; local url_parse = require "socket.url".parse; local urldecode = require "util.http".urldecode; +local dbuffer = require "util.dbuffer"; local function preprocess_path(path) path = urldecode((path:gsub("//+", "/"))); @@ -28,10 +28,13 @@ local httpstream = {}; function httpstream.new(success_cb, error_cb, parser_type, options_cb) local client = true; if not parser_type or parser_type == "server" then client = false; else assert(parser_type == "client", "Invalid parser type"); end - local buf, buflen, buftable = {}, 0, true; local bodylimit = tonumber(options_cb and options_cb().body_size_limit) or 10*1024*1024; + -- https://stackoverflow.com/a/686243 + -- Indiviual headers can be up to 16k? What madness? + local headlimit = tonumber(options_cb and options_cb().head_size_limit) or 10*1024; local buflimit = tonumber(options_cb and options_cb().buffer_size_limit) or bodylimit * 2; - local chunked, chunk_size, chunk_start; + local buffer = dbuffer.new(buflimit); + local chunked; local state = nil; local packet; local len; @@ -41,32 +44,26 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) feed = function(_, data) if error then return nil, "parse has failed"; end if not data then -- EOF - if buftable then buf, buftable = t_concat(buf), false; end if state and client and not len then -- reading client body until EOF - packet.body = buf; + buffer:collapse(); + packet.body = buffer:read_chunk() or ""; success_cb(packet); - elseif buf ~= "" then -- unexpected EOF + state = nil; + elseif buffer:length() ~= 0 then -- unexpected EOF error = true; return error_cb("unexpected-eof"); end return; end - if buftable then - t_insert(buf, data); - else - buf = { buf, data }; - buftable = true; - end - buflen = buflen + #data; - if buflen > buflimit then error = true; return error_cb("max-buffer-size-exceeded"); end - while buflen > 0 do + if not buffer:write(data) then error = true; return error_cb("max-buffer-size-exceeded"); end + while buffer:length() > 0 do if state == nil then -- read request - if buftable then buf, buftable = t_concat(buf), false; end - local index = buf:find("\r\n\r\n", nil, true); + local index = buffer:sub(1, headlimit):find("\r\n\r\n", nil, true); if not index then return; end -- not enough data - local method, path, httpversion, status_code, reason_phrase; + -- FIXME was reason_phrase meant to be passed on somewhere? + local method, path, httpversion, status_code, reason_phrase; -- luacheck: ignore reason_phrase local first_line; local headers = {}; - for line in buf:sub(1,index+1):gmatch("([^\r\n]+)\r\n") do -- parse request + for line in buffer:read(index+3):gmatch("([^\r\n]+)\r\n") do -- parse request if first_line then local key, val = line:match("^([^%s:]+): *(.*)$"); if not key then error = true; return error_cb("invalid-header-line"); end -- TODO handle multi-line and invalid headers @@ -91,7 +88,6 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) if not first_line then error = true; return error_cb("invalid-status-line"); end chunked = have_body and headers["transfer-encoding"] == "chunked"; len = tonumber(headers["content-length"]); -- TODO check for invalid len - if len and len > bodylimit then error = true; return error_cb("content-length-limit-exceeded"); end if client then -- FIXME handle '100 Continue' response (by skipping it) if not have_body then len = 0; end @@ -99,7 +95,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) code = status_code; httpversion = httpversion; headers = headers; - body = have_body and "" or nil; + body = false; -- COMPAT the properties below are deprecated responseversion = httpversion; responseheaders = headers; @@ -124,60 +120,72 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) path = path; httpversion = httpversion; headers = headers; - body = nil; + body = false; + body_sink = nil; }; end - buf = buf:sub(index + 4); - buflen = #buf; + if len and len > bodylimit then + -- Early notification, for redirection + success_cb(packet); + if not packet.body_sink then error = true; return error_cb("content-length-limit-exceeded"); end + end + if chunked and not packet.body_sink then + success_cb(packet); + if not packet.body_sink then + packet.body_buffer = dbuffer.new(buflimit); + end + end state = true; end if state then -- read body - if client then - if chunked then - if chunk_start and buflen - chunk_start - 2 < chunk_size then - return; - end -- not enough data - if buftable then buf, buftable = t_concat(buf), false; end - if not buf:find("\r\n", nil, true) then - return; - end -- not enough data - if not chunk_size then - chunk_size, chunk_start = buf:match("^(%x+)[^\r\n]*\r\n()"); - chunk_size = chunk_size and tonumber(chunk_size, 16); - if not chunk_size then error = true; return error_cb("invalid-chunk-size"); end - end - if chunk_size == 0 and buf:find("\r\n\r\n", chunk_start-2, true) then - state, chunk_size = nil, nil; - buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped - success_cb(packet); - elseif buflen - chunk_start - 2 >= chunk_size then -- we have a chunk - packet.body = packet.body..buf:sub(chunk_start, chunk_start + (chunk_size-1)); - buf = buf:sub(chunk_start + chunk_size + 2); - buflen = buflen - (chunk_start + chunk_size + 2 - 1); - chunk_size, chunk_start = nil, nil; - else -- Partial chunk remaining - break; + if chunked then + local chunk_header = buffer:sub(1, 512); -- XXX How large do chunk headers grow? + local chunk_size, chunk_start = chunk_header:match("^(%x+)[^\r\n]*\r\n()"); + if not chunk_size then return; end + chunk_size = chunk_size and tonumber(chunk_size, 16); + if not chunk_size then error = true; return error_cb("invalid-chunk-size"); end + if chunk_size == 0 and chunk_header:find("\r\n\r\n", chunk_start-2, true) then + local body_buffer = packet.body_buffer; + if body_buffer then + packet.body_buffer = nil; + body_buffer:collapse(); + packet.body = body_buffer:read_chunk() or ""; end - elseif len and buflen >= len then - if buftable then buf, buftable = t_concat(buf), false; end - if packet.code == 101 then - packet.body, buf, buflen, buftable = buf, {}, 0, true; + + buffer:collapse(); + local buf = buffer:read_chunk(); + buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped + buffer:write(buf); + state, chunked = nil, nil; + success_cb(packet); + elseif buffer:length() - chunk_start - 2 >= chunk_size then -- we have a chunk + buffer:discard(chunk_start - 1); -- TODO verify that it's not off-by-one + (packet.body_sink or packet.body_buffer):write(buffer:read(chunk_size)); + buffer:discard(2); -- CRLF + else -- Partial chunk remaining + break; + end + elseif packet.body_sink then + local chunk = buffer:read_chunk(len); + while chunk and len > 0 do + if packet.body_sink:write(chunk) then + len = len - #chunk; + chunk = buffer:read_chunk(len); else - packet.body, buf = buf:sub(1, len), buf:sub(len + 1); - buflen = #buf; + error = true; + return error_cb("body-sink-write-failure"); end - state = nil; success_cb(packet); - else - break; end - elseif buflen >= len then - if buftable then buf, buftable = t_concat(buf), false; end - packet.body, buf = buf:sub(1, len), buf:sub(len + 1); - buflen = #buf; + if len == 0 then state = nil; success_cb(packet); end + elseif buffer:length() >= len then + assert(not chunked) + packet.body = buffer:read(len) or ""; state = nil; success_cb(packet); else break; end + else + break; end end end; diff --git a/net/http/server.lua b/net/http/server.lua index 18704962..0070367a 100644 --- a/net/http/server.lua +++ b/net/http/server.lua @@ -13,6 +13,8 @@ local traceback = debug.traceback; local tostring = tostring; local cache = require "util.cache"; local codes = require "net.http.codes"; +local promise = require "util.promise"; +local errors = require "util.error"; local blocksize = 2^16; local _M = {}; @@ -170,6 +172,49 @@ local headerfix = setmetatable({}, { end }); +local function handle_result(request, response, result) + if result == nil then + result = 404; + end + + if result == true then + return; + end + + local body; + local result_type = type(result); + if result_type == "number" then + response.status_code = result; + if result >= 400 then + body = events.fire_event("http-error", { request = request, response = response, code = result }); + end + elseif result_type == "string" then + body = result; + elseif errors.is_err(result) then + response.status_code = result.code or 500; + body = events.fire_event("http-error", { request = request, response = response, code = result.code or 500, error = result }); + elseif promise.is_promise(result) then + result:next(function (ret) + handle_result(request, response, ret); + end, function (err) + response.status_code = 500; + handle_result(request, response, err or 500); + end); + return true; + elseif result_type == "table" then + for k, v in pairs(result) do + if k ~= "headers" then + response[k] = v; + else + for header_name, header_value in pairs(v) do + response.headers[header_name] = header_value; + end + end + end + end + return response:send(body); +end + function _M.hijack_response(response, listener) -- luacheck: ignore error("TODO"); end @@ -194,8 +239,11 @@ function handle_request(conn, request, finish_cb) response_conn_header = httpversion == "1.1" and "close" or nil end + local is_head_request = request.method == "HEAD"; + local response = { request = request; + is_head_request = is_head_request; status_code = 200; headers = { date = date_header, connection = response_conn_header }; persistent = persistent; @@ -227,6 +275,11 @@ function handle_request(conn, request, finish_cb) local payload = { request = request, response = response }; log("debug", "Firing event: %s", global_event); local result = events.fire_event(global_event, payload); + if result == nil and is_head_request then + local global_head_event = "GET "..request.path:match("[^?]*"); + log("debug", "Firing event: %s", global_head_event); + result = events.fire_event(global_head_event, payload); + end if result == nil then if not hosts[host] then if hosts[default_host] then @@ -247,40 +300,17 @@ function handle_request(conn, request, finish_cb) local host_event = request.method.." "..host..request.path:match("[^?]*"); log("debug", "Firing event: %s", host_event); result = events.fire_event(host_event, payload); - end - if result ~= nil then - if result ~= true then - local body; - local result_type = type(result); - if result_type == "number" then - response.status_code = result; - if result >= 400 then - payload.code = result; - body = events.fire_event("http-error", payload); - end - elseif result_type == "string" then - body = result; - elseif result_type == "table" then - for k, v in pairs(result) do - if k ~= "headers" then - response[k] = v; - else - for header_name, header_value in pairs(v) do - response.headers[header_name] = header_value; - end - end - end - end - response:send(body); + + if result == nil and is_head_request then + local host_head_event = "GET "..host..request.path:match("[^?]*"); + log("debug", "Firing event: %s", host_head_event); + result = events.fire_event(host_head_event, payload); end - return; end - -- if handler not called, return 404 - response.status_code = 404; - payload.code = 404; - response:send(events.fire_event("http-error", payload)); + return handle_result(request, response, result); end + local function prepare_header(response) local status_line = "HTTP/"..response.request.httpversion.." "..(response.status or codes[response.status_code]); local headers = response.headers; @@ -292,16 +322,29 @@ local function prepare_header(response) return output; end _M.prepare_header = prepare_header; +function _M.send_head_response(response) + if response.finished then return; end + local output = prepare_header(response); + response.conn:write(t_concat(output)); + response:done(); +end function _M.send_response(response, body) if response.finished then return; end body = body or response.body or ""; - response.headers.content_length = #body; + response.headers.content_length = ("%d"):format(#body); + if response.is_head_request then + return _M.send_head_response(response) + end local output = prepare_header(response); t_insert(output, body); response.conn:write(t_concat(output)); response:done(); end function _M.send_file(response, f) + if response.is_head_request then + if f.close then f:close(); end + return _M.send_head_response(response); + end if response.finished then return; end local chunked = not response.headers.content_length; if chunked then response.headers.transfer_encoding = "chunked"; end |