From 6f6ac910564bf6ecdc0e6b70cf06bd84a24868fb Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 5 Apr 2019 16:10:51 +0200 Subject: net.http.files: Copy of mod_http_files The intent is to make it easier to reuse and simplify mod_http_files. Currently modules will use the serve() function exported by mod_http_files in order to serve their own files. This makes it unclear whether mod_http_files should be doing anything on its own. Moving the logic into a separate module should help here, as well as make re-use outside of prosody easier. --- net/http/files.lua | 198 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 net/http/files.lua (limited to 'net/http') diff --git a/net/http/files.lua b/net/http/files.lua new file mode 100644 index 00000000..1dae0d6d --- /dev/null +++ b/net/http/files.lua @@ -0,0 +1,198 @@ +-- Prosody IM +-- Copyright (C) 2008-2010 Matthew Wild +-- Copyright (C) 2008-2010 Waqas Hussain +-- +-- This project is MIT/X11 licensed. Please see the +-- COPYING file in the source package for more information. +-- + +module:depends("http"); +local server = require"net.http.server"; +local lfs = require "lfs"; + +local os_date = os.date; +local open = io.open; +local stat = lfs.attributes; +local build_path = require"socket.url".build_path; +local path_sep = package.config:sub(1,1); + +local base_path = module:get_option_path("http_files_dir", module:get_option_path("http_path")); +local cache_size = module:get_option_number("http_files_cache_size", 128); +local cache_max_file_size = module:get_option_number("http_files_cache_max_file_size", 4096); +local dir_indices = module:get_option_array("http_index_files", { "index.html", "index.htm" }); +local directory_index = module:get_option_boolean("http_dir_listing"); + +local mime_map = module:shared("/*/http_files/mime").types; +if not mime_map then + mime_map = { + html = "text/html", htm = "text/html", + xml = "application/xml", + txt = "text/plain", + css = "text/css", + js = "application/javascript", + png = "image/png", + gif = "image/gif", + jpeg = "image/jpeg", jpg = "image/jpeg", + svg = "image/svg+xml", + }; + module:shared("/*/http_files/mime").types = mime_map; + + local mime_types, err = open(module:get_option_path("mime_types_file", "/etc/mime.types", "config"), "r"); + if mime_types then + local mime_data = mime_types:read("*a"); + mime_types:close(); + setmetatable(mime_map, { + __index = function(t, ext) + local typ = mime_data:match("\n(%S+)[^\n]*%s"..(ext:lower()).."%s") or "application/octet-stream"; + t[ext] = typ; + return typ; + end + }); + end +end + +local forbidden_chars_pattern = "[/%z]"; +if prosody.platform == "windows" then + forbidden_chars_pattern = "[/%z\001-\031\127\"*:<>?|]" +end + +local urldecode = require "util.http".urldecode; +function sanitize_path(path) + if not path then return end + local out = {}; + + local c = 0; + for component in path:gmatch("([^/]+)") do + component = urldecode(component); + if component:find(forbidden_chars_pattern) then + return nil; + elseif component == ".." then + if c <= 0 then + return nil; + end + out[c] = nil; + c = c - 1; + elseif component ~= "." then + c = c + 1; + out[c] = component; + end + end + if path:sub(-1,-1) == "/" then + out[c+1] = ""; + end + return "/"..table.concat(out, "/"); +end + +local cache = require "util.cache".new(cache_size); + +function serve(opts) + if type(opts) ~= "table" then -- assume path string + opts = { path = opts }; + end + -- luacheck: ignore 431 + local base_path = opts.path; + local dir_indices = opts.index_files or dir_indices; + local directory_index = opts.directory_index; + local function serve_file(event, path) + local request, response = event.request, event.response; + local sanitized_path = sanitize_path(path); + if path and not sanitized_path then + return 400; + end + path = sanitized_path; + local orig_path = sanitize_path(request.path); + local full_path = base_path .. (path or ""):gsub("/", path_sep); + local attr = stat(full_path:match("^.*[^\\/]")); -- Strip trailing path separator because Windows + if not attr then + return 404; + end + + local request_headers, response_headers = request.headers, response.headers; + + local last_modified = os_date('!%a, %d %b %Y %H:%M:%S GMT', attr.modification); + response_headers.last_modified = last_modified; + + local etag = ('"%02x-%x-%x-%x"'):format(attr.dev or 0, attr.ino or 0, attr.size or 0, attr.modification or 0); + response_headers.etag = etag; + + local if_none_match = request_headers.if_none_match + local if_modified_since = request_headers.if_modified_since; + if etag == if_none_match + or (not if_none_match and last_modified == if_modified_since) then + return 304; + end + + local data = cache:get(orig_path); + if data and data.etag == etag then + response_headers.content_type = data.content_type; + data = data.data; + elseif attr.mode == "directory" and path then + if full_path:sub(-1) ~= "/" then + local dir_path = { is_absolute = true, is_directory = true }; + for dir in orig_path:gmatch("[^/]+") do dir_path[#dir_path+1]=dir; end + response_headers.location = build_path(dir_path); + return 301; + end + for i=1,#dir_indices do + if stat(full_path..dir_indices[i], "mode") == "file" then + return serve_file(event, path..dir_indices[i]); + end + end + + if directory_index then + data = server._events.fire_event("directory-index", { path = request.path, full_path = full_path }); + end + if not data then + return 403; + end + cache:set(orig_path, { data = data, content_type = mime_map.html; etag = etag; }); + response_headers.content_type = mime_map.html; + + else + local f, err = open(full_path, "rb"); + if not f then + module:log("debug", "Could not open %s. Error was %s", full_path, err); + return 403; + end + local ext = full_path:match("%.([^./]+)$"); + local content_type = ext and mime_map[ext]; + response_headers.content_type = content_type; + if attr.size > cache_max_file_size then + response_headers.content_length = attr.size; + module:log("debug", "%d > cache_max_file_size", attr.size); + return response:send_file(f); + else + data = f:read("*a"); + f:close(); + end + cache:set(orig_path, { data = data; content_type = content_type; etag = etag }); + end + + return response:send(data); + end + + return serve_file; +end + +function wrap_route(routes) + for route,handler in pairs(routes) do + if type(handler) ~= "function" then + routes[route] = serve(handler); + end + end + return routes; +end + +if base_path then + module:provides("http", { + route = { + ["GET /*"] = serve { + path = base_path; + directory_index = directory_index; + } + }; + }); +else + module:log("debug", "http_files_dir not set, assuming use by some other module"); +end + -- cgit v1.2.3 From 3ea6ca719574ce8a1bfa9532c5526da05f3ff5d9 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 5 Apr 2019 17:09:03 +0200 Subject: net.http.files: Make into standalone library --- net/http/files.lua | 78 ++++++++++-------------------------------------------- 1 file changed, 14 insertions(+), 64 deletions(-) (limited to 'net/http') diff --git a/net/http/files.lua b/net/http/files.lua index 1dae0d6d..0b898dc6 100644 --- a/net/http/files.lua +++ b/net/http/files.lua @@ -6,9 +6,10 @@ -- COPYING file in the source package for more information. -- -module:depends("http"); local server = require"net.http.server"; local lfs = require "lfs"; +local new_cache = require "util.cache".new; +local log = require "util.logger".init("net.http.files"); local os_date = os.date; local open = io.open; @@ -16,48 +17,14 @@ local stat = lfs.attributes; local build_path = require"socket.url".build_path; local path_sep = package.config:sub(1,1); -local base_path = module:get_option_path("http_files_dir", module:get_option_path("http_path")); -local cache_size = module:get_option_number("http_files_cache_size", 128); -local cache_max_file_size = module:get_option_number("http_files_cache_max_file_size", 4096); -local dir_indices = module:get_option_array("http_index_files", { "index.html", "index.htm" }); -local directory_index = module:get_option_boolean("http_dir_listing"); - -local mime_map = module:shared("/*/http_files/mime").types; -if not mime_map then - mime_map = { - html = "text/html", htm = "text/html", - xml = "application/xml", - txt = "text/plain", - css = "text/css", - js = "application/javascript", - png = "image/png", - gif = "image/gif", - jpeg = "image/jpeg", jpg = "image/jpeg", - svg = "image/svg+xml", - }; - module:shared("/*/http_files/mime").types = mime_map; - - local mime_types, err = open(module:get_option_path("mime_types_file", "/etc/mime.types", "config"), "r"); - if mime_types then - local mime_data = mime_types:read("*a"); - mime_types:close(); - setmetatable(mime_map, { - __index = function(t, ext) - local typ = mime_data:match("\n(%S+)[^\n]*%s"..(ext:lower()).."%s") or "application/octet-stream"; - t[ext] = typ; - return typ; - end - }); - end -end local forbidden_chars_pattern = "[/%z]"; -if prosody.platform == "windows" then +if package.config:sub(1,1) == "\\" then forbidden_chars_pattern = "[/%z\001-\031\127\"*:<>?|]" end local urldecode = require "util.http".urldecode; -function sanitize_path(path) +local function sanitize_path(path) --> util.paths or util.http? if not path then return end local out = {}; @@ -83,15 +50,16 @@ function sanitize_path(path) return "/"..table.concat(out, "/"); end -local cache = require "util.cache".new(cache_size); - -function serve(opts) +local function serve(opts) if type(opts) ~= "table" then -- assume path string opts = { path = opts }; end + local mime_map = opts.mime_map or { html = "text/html" }; + local cache = new_cache(opts.cache_size or 256); + local cache_max_file_size = tonumber(opts.cache_max_file_size) or 1024 -- luacheck: ignore 431 local base_path = opts.path; - local dir_indices = opts.index_files or dir_indices; + local dir_indices = opts.index_files or { "index.html", "index.htm" }; local directory_index = opts.directory_index; local function serve_file(event, path) local request, response = event.request, event.response; @@ -151,7 +119,7 @@ function serve(opts) else local f, err = open(full_path, "rb"); if not f then - module:log("debug", "Could not open %s. Error was %s", full_path, err); + log("debug", "Could not open %s. Error was %s", full_path, err); return 403; end local ext = full_path:match("%.([^./]+)$"); @@ -159,7 +127,7 @@ function serve(opts) response_headers.content_type = content_type; if attr.size > cache_max_file_size then response_headers.content_length = attr.size; - module:log("debug", "%d > cache_max_file_size", attr.size); + log("debug", "%d > cache_max_file_size", attr.size); return response:send_file(f); else data = f:read("*a"); @@ -174,25 +142,7 @@ function serve(opts) return serve_file; end -function wrap_route(routes) - for route,handler in pairs(routes) do - if type(handler) ~= "function" then - routes[route] = serve(handler); - end - end - return routes; -end - -if base_path then - module:provides("http", { - route = { - ["GET /*"] = serve { - path = base_path; - directory_index = directory_index; - } - }; - }); -else - module:log("debug", "http_files_dir not set, assuming use by some other module"); -end +return { + serve = serve; +} -- cgit v1.2.3 From a371d01137d44a43cbeabe2590174fc36053bb2a Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 4 May 2019 04:48:40 +0200 Subject: net.http.files: Bump cache hits so they stay cached It's not an LRU cache unless this is done. --- net/http/files.lua | 1 + 1 file changed, 1 insertion(+) (limited to 'net/http') diff --git a/net/http/files.lua b/net/http/files.lua index 0b898dc6..090b15c8 100644 --- a/net/http/files.lua +++ b/net/http/files.lua @@ -94,6 +94,7 @@ local function serve(opts) if data and data.etag == etag then response_headers.content_type = data.content_type; data = data.data; + cache:get(orig_path, data); elseif attr.mode == "directory" and path then if full_path:sub(-1) ~= "/" then local dir_path = { is_absolute = true, is_directory = true }; -- cgit v1.2.3 From c6ca3b473e60f56a20113a3ab9d8a33231aae0e5 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 29 Jun 2019 19:19:38 +0200 Subject: net.http.files: Fix cache handling Typo that broke the LRU-ness of the caching --- net/http/files.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'net/http') diff --git a/net/http/files.lua b/net/http/files.lua index 090b15c8..7ff81fc8 100644 --- a/net/http/files.lua +++ b/net/http/files.lua @@ -94,7 +94,7 @@ local function serve(opts) if data and data.etag == etag then response_headers.content_type = data.content_type; data = data.data; - cache:get(orig_path, data); + cache:set(orig_path, data); elseif attr.mode == "directory" and path then if full_path:sub(-1) ~= "/" then local dir_path = { is_absolute = true, is_directory = true }; -- cgit v1.2.3 From 89a6f8d8c108bdf633d7d6589c260de833c56bf5 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 12 Oct 2019 18:27:02 +0200 Subject: net.http.server: Ensure HEAD requests are sent with empty body --- net/http/server.lua | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'net/http') diff --git a/net/http/server.lua b/net/http/server.lua index 9b63d516..47680455 100644 --- a/net/http/server.lua +++ b/net/http/server.lua @@ -194,8 +194,11 @@ function handle_request(conn, request, finish_cb) response_conn_header = httpversion == "1.1" and "close" or nil end + local is_head_request = request.method == "HEAD"; + local response = { request = request; + is_head_request = is_head_request; status_code = 200; headers = { date = date_header, connection = response_conn_header }; persistent = persistent; @@ -291,16 +294,29 @@ local function prepare_header(response) return output; end _M.prepare_header = prepare_header; +function _M.send_head_response(response) + if response.finished then return; end + local output = prepare_header(response); + response.conn:write(t_concat(output)); + response:done(); +end function _M.send_response(response, body) if response.finished then return; end body = body or response.body or ""; response.headers.content_length = #body; + if response.is_head_request then + return _M.send_head_response(response) + end local output = prepare_header(response); t_insert(output, body); response.conn:write(t_concat(output)); response:done(); end function _M.send_file(response, f) + if response.is_head_request then + if f.close then f:close(); end + return _M.send_head_response(response); + end if response.finished then return; end local chunked = not response.headers.content_length; if chunked then response.headers.transfer_encoding = "chunked"; end -- cgit v1.2.3 From 8e485ec3200359adcc00731a371b4455d24bb562 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 12 Oct 2019 18:27:54 +0200 Subject: net.http.server: Re-fire unhandled HEAD requsts as GET events (fixes #1447) BC: This overloads the GET event. Previous commit ensures HEAD requests are sent without a body. --- net/http/server.lua | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'net/http') diff --git a/net/http/server.lua b/net/http/server.lua index 47680455..b649ff5f 100644 --- a/net/http/server.lua +++ b/net/http/server.lua @@ -229,6 +229,11 @@ function handle_request(conn, request, finish_cb) local payload = { request = request, response = response }; log("debug", "Firing event: %s", global_event); local result = events.fire_event(global_event, payload); + if result == nil and is_head_request then + local global_head_event = "GET "..request.path:match("[^?]*"); + log("debug", "Firing event: %s", global_head_event); + result = events.fire_event(global_head_event, payload); + end if result == nil then if not hosts[host] then if hosts[default_host] then @@ -249,6 +254,12 @@ function handle_request(conn, request, finish_cb) local host_event = request.method.." "..host..request.path:match("[^?]*"); log("debug", "Firing event: %s", host_event); result = events.fire_event(host_event, payload); + + if result == nil and is_head_request then + local host_head_event = "GET "..host..request.path:match("[^?]*"); + log("debug", "Firing event: %s", host_head_event); + result = events.fire_event(host_head_event, payload); + end end if result ~= nil then if result ~= true then -- cgit v1.2.3 From 30a72c72a392fdbee1e91946c9a72ed88a20b232 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 12 Oct 2019 19:30:29 +0200 Subject: net.http.server: Explicitly convert number to string, avoiding implicit coercion --- net/http/server.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'net/http') diff --git a/net/http/server.lua b/net/http/server.lua index b649ff5f..bf24c97e 100644 --- a/net/http/server.lua +++ b/net/http/server.lua @@ -314,7 +314,7 @@ end function _M.send_response(response, body) if response.finished then return; end body = body or response.body or ""; - response.headers.content_length = #body; + response.headers.content_length = ("%d"):format(#body); if response.is_head_request then return _M.send_head_response(response) end -- cgit v1.2.3 From 9e6dce07bf7a9836eb8123bf198c8671d423157f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 12 Oct 2019 19:31:48 +0200 Subject: net.http.files: Explicitly convert number to string, avoiding implicit coercion --- net/http/files.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'net/http') diff --git a/net/http/files.lua b/net/http/files.lua index 7ff81fc8..650c6f47 100644 --- a/net/http/files.lua +++ b/net/http/files.lua @@ -127,7 +127,7 @@ local function serve(opts) local content_type = ext and mime_map[ext]; response_headers.content_type = content_type; if attr.size > cache_max_file_size then - response_headers.content_length = attr.size; + response_headers.content_length = ("%d"):format(attr.size); log("debug", "%d > cache_max_file_size", attr.size); return response:send_file(f); else -- cgit v1.2.3 From 2b035680860450dd6ce9d3b853c26bd900752f8f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 1 Nov 2019 23:18:29 +0100 Subject: net.http.codes: Avoid implicit number -> string coercion --- net/http/codes.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'net/http') diff --git a/net/http/codes.lua b/net/http/codes.lua index 8098b5c3..4327f151 100644 --- a/net/http/codes.lua +++ b/net/http/codes.lua @@ -82,5 +82,5 @@ local response_codes = { -- [512-599] = "Unassigned"; }; -for k,v in pairs(response_codes) do response_codes[k] = k.." "..v; end +for k,v in pairs(response_codes) do response_codes[k] = ("%03d %s"):format(k, v); end return setmetatable(response_codes, { __index = function(_, k) return k.." Unassigned"; end }) -- cgit v1.2.3 From 6a73014b9acdf69a7c415ac8f5fa173f83a68362 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 1 Nov 2019 22:25:54 +0100 Subject: net.http.server: Factor out handling of event response for easier reuse --- net/http/server.lua | 65 ++++++++++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 31 deletions(-) (limited to 'net/http') diff --git a/net/http/server.lua b/net/http/server.lua index bf24c97e..991ef970 100644 --- a/net/http/server.lua +++ b/net/http/server.lua @@ -170,6 +170,38 @@ local headerfix = setmetatable({}, { end }); +local function handle_result(request, response, result) + if result == nil then + result = 404; + end + + if result == true then + return; + end + + local body; + local result_type = type(result); + if result_type == "number" then + response.status_code = result; + if result >= 400 then + body = events.fire_event("http-error", { request = request, response = response, code = result }); + end + elseif result_type == "string" then + body = result; + elseif result_type == "table" then + for k, v in pairs(result) do + if k ~= "headers" then + response[k] = v; + else + for header_name, header_value in pairs(v) do + response.headers[header_name] = header_value; + end + end + end + end + response:send(body); +end + function _M.hijack_response(response, listener) -- luacheck: ignore error("TODO"); end @@ -261,39 +293,10 @@ function handle_request(conn, request, finish_cb) result = events.fire_event(host_head_event, payload); end end - if result ~= nil then - if result ~= true then - local body; - local result_type = type(result); - if result_type == "number" then - response.status_code = result; - if result >= 400 then - payload.code = result; - body = events.fire_event("http-error", payload); - end - elseif result_type == "string" then - body = result; - elseif result_type == "table" then - for k, v in pairs(result) do - if k ~= "headers" then - response[k] = v; - else - for header_name, header_value in pairs(v) do - response.headers[header_name] = header_value; - end - end - end - end - response:send(body); - end - return; - end - -- if handler not called, return 404 - response.status_code = 404; - payload.code = 404; - response:send(events.fire_event("http-error", payload)); + return handle_result(request, response, result); end + local function prepare_header(response) local status_line = "HTTP/"..response.request.httpversion.." "..(response.status or codes[response.status_code]); local headers = response.headers; -- cgit v1.2.3 From fbab8ed06ac738e6aec96db67141f987b9d3c058 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 1 Nov 2019 22:28:39 +0100 Subject: net.http.server: Tail call because tail call! --- net/http/server.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'net/http') diff --git a/net/http/server.lua b/net/http/server.lua index 991ef970..0682d6ed 100644 --- a/net/http/server.lua +++ b/net/http/server.lua @@ -199,7 +199,7 @@ local function handle_result(request, response, result) end end end - response:send(body); + return response:send(body); end function _M.hijack_response(response, listener) -- luacheck: ignore -- cgit v1.2.3 From fdfe1e0b2bf8cea4e2e512fa8e25bface008a59b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 1 Nov 2019 22:30:35 +0100 Subject: net.http.server: Handle util.error objects from http handlers --- net/http/server.lua | 3 +++ 1 file changed, 3 insertions(+) (limited to 'net/http') diff --git a/net/http/server.lua b/net/http/server.lua index 0682d6ed..0d49dbce 100644 --- a/net/http/server.lua +++ b/net/http/server.lua @@ -13,6 +13,7 @@ local traceback = debug.traceback; local tostring = tostring; local cache = require "util.cache"; local codes = require "net.http.codes"; +local errors = require "util.error"; local blocksize = 2^16; local _M = {}; @@ -188,6 +189,8 @@ local function handle_result(request, response, result) end elseif result_type == "string" then body = result; + elseif errors.is_err(result) then + body = events.fire_event("http-error", { request = request, response = response, code = result.code, error = result }); elseif result_type == "table" then for k, v in pairs(result) do if k ~= "headers" then -- cgit v1.2.3 From 5cac177270b40336e9820619e9c6ab3973ad95b4 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 1 Nov 2019 22:31:15 +0100 Subject: net.http.server: Handle promises from http handlers --- net/http/server.lua | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'net/http') diff --git a/net/http/server.lua b/net/http/server.lua index 0d49dbce..dc64728f 100644 --- a/net/http/server.lua +++ b/net/http/server.lua @@ -13,6 +13,7 @@ local traceback = debug.traceback; local tostring = tostring; local cache = require "util.cache"; local codes = require "net.http.codes"; +local promise = require "util.promise"; local errors = require "util.error"; local blocksize = 2^16; @@ -191,6 +192,13 @@ local function handle_result(request, response, result) body = result; elseif errors.is_err(result) then body = events.fire_event("http-error", { request = request, response = response, code = result.code, error = result }); + elseif promise.is_promise(result) then + result:next(function (ret) + handle_result(request, response, ret); + end, function (err) + handle_result(request, response, err); + end); + return true; elseif result_type == "table" then for k, v in pairs(result) do if k ~= "headers" then -- cgit v1.2.3 From 58990598f283f24f99dd5899669acc310b5437b1 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 5 Nov 2019 01:34:13 +0100 Subject: net.http.server: Treat promise rejection without value as a HTTP 500 error --- net/http/server.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'net/http') diff --git a/net/http/server.lua b/net/http/server.lua index dc64728f..47f064a5 100644 --- a/net/http/server.lua +++ b/net/http/server.lua @@ -196,7 +196,7 @@ local function handle_result(request, response, result) result:next(function (ret) handle_result(request, response, ret); end, function (err) - handle_result(request, response, err); + handle_result(request, response, err or 500); end); return true; elseif result_type == "table" then -- cgit v1.2.3 From 87d0125802ce38410e5d429c90760802dec0397c Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 14 Dec 2019 20:28:44 +0100 Subject: util.error: Move default for numeric error code to net.http.server Stanza errors can also have numbers but these are a legacy thing and rarely used, except in MUC. HTTP errors on the other hand always have a number. --- net/http/server.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'net/http') diff --git a/net/http/server.lua b/net/http/server.lua index 47f064a5..f4f67d18 100644 --- a/net/http/server.lua +++ b/net/http/server.lua @@ -191,7 +191,7 @@ local function handle_result(request, response, result) elseif result_type == "string" then body = result; elseif errors.is_err(result) then - body = events.fire_event("http-error", { request = request, response = response, code = result.code, error = result }); + body = events.fire_event("http-error", { request = request, response = response, code = result.code or 500, error = result }); elseif promise.is_promise(result) then result:next(function (ret) handle_result(request, response, ret); -- cgit v1.2.3 From 248ec3f83401528152c3b34dd81e994563706fea Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 23 Dec 2019 21:29:34 +0100 Subject: net.http.parser: Silence warning about unused variable [luacheck] --- net/http/parser.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'net/http') diff --git a/net/http/parser.lua b/net/http/parser.lua index 4e4ae9fb..62c09aef 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -63,7 +63,8 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) if buftable then buf, buftable = t_concat(buf), false; end local index = buf:find("\r\n\r\n", nil, true); if not index then return; end -- not enough data - local method, path, httpversion, status_code, reason_phrase; + -- FIXME was reason_phrase meant to be passed on somewhere? + local method, path, httpversion, status_code, reason_phrase; -- luacheck: ignore reason_phrase local first_line; local headers = {}; for line in buf:sub(1,index+1):gmatch("([^\r\n]+)\r\n") do -- parse request -- cgit v1.2.3 From 48bb417812710633ce1340e1f0f91366905b2a4f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 1 Jan 2020 01:22:57 +0100 Subject: net.http.parser: Add TODO related to #726 --- net/http/parser.lua | 1 + 1 file changed, 1 insertion(+) (limited to 'net/http') diff --git a/net/http/parser.lua b/net/http/parser.lua index 62c09aef..b328bdfe 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -93,6 +93,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) chunked = have_body and headers["transfer-encoding"] == "chunked"; len = tonumber(headers["content-length"]); -- TODO check for invalid len if len and len > bodylimit then error = true; return error_cb("content-length-limit-exceeded"); end + -- TODO ask a callback whether to proceed in case of large requests or Expect: 100-continue if client then -- FIXME handle '100 Continue' response (by skipping it) if not have_body then len = 0; end -- cgit v1.2.3 From 4f2548e8efbb5659ed723cb4e802ef66328ada29 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 12 Apr 2020 22:57:14 +0200 Subject: net.http.server: Use error code from util.error (fixes #1502) Oversight in 955e54e451dc when this was added. --- net/http/server.lua | 1 + 1 file changed, 1 insertion(+) (limited to 'net/http') diff --git a/net/http/server.lua b/net/http/server.lua index f4f67d18..c59479cd 100644 --- a/net/http/server.lua +++ b/net/http/server.lua @@ -191,6 +191,7 @@ local function handle_result(request, response, result) elseif result_type == "string" then body = result; elseif errors.is_err(result) then + response.status_code = result.code or 500; body = events.fire_event("http-error", { request = request, response = response, code = result.code or 500, error = result }); elseif promise.is_promise(result) then result:next(function (ret) -- cgit v1.2.3 From 64aa6a2a0e704221821cb53ccd90e19fabfa475e Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 1 Aug 2020 18:14:09 +0200 Subject: net.http.parser: Switch to util.dbuffer for buffering incoming data This is primarily a step towards saving uploads directly to files, tho this should hopefully be more efficient than collapsing the entire buffer to a single string every now and then. --- net/http/parser.lua | 110 +++++++++++++++++++++++----------------------------- 1 file changed, 49 insertions(+), 61 deletions(-) (limited to 'net/http') diff --git a/net/http/parser.lua b/net/http/parser.lua index b328bdfe..84b1e005 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -1,8 +1,8 @@ local tonumber = tonumber; local assert = assert; -local t_insert, t_concat = table.insert, table.concat; local url_parse = require "socket.url".parse; local urldecode = require "util.http".urldecode; +local dbuffer = require "util.dbuffer"; local function preprocess_path(path) path = urldecode((path:gsub("//+", "/"))); @@ -28,10 +28,13 @@ local httpstream = {}; function httpstream.new(success_cb, error_cb, parser_type, options_cb) local client = true; if not parser_type or parser_type == "server" then client = false; else assert(parser_type == "client", "Invalid parser type"); end - local buf, buflen, buftable = {}, 0, true; local bodylimit = tonumber(options_cb and options_cb().body_size_limit) or 10*1024*1024; + -- https://stackoverflow.com/a/686243 + -- Indiviual headers can be up to 16k? What madness? + local headlimit = tonumber(options_cb and options_cb().head_size_limit) or 10*1024; local buflimit = tonumber(options_cb and options_cb().buffer_size_limit) or bodylimit * 2; - local chunked, chunk_size, chunk_start; + local buffer = dbuffer.new(buflimit); + local chunked; local state = nil; local packet; local len; @@ -41,33 +44,26 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) feed = function(_, data) if error then return nil, "parse has failed"; end if not data then -- EOF - if buftable then buf, buftable = t_concat(buf), false; end if state and client and not len then -- reading client body until EOF - packet.body = buf; + buffer:collapse(); + packet.body = buffer:read_chunk() or ""; success_cb(packet); - elseif buf ~= "" then -- unexpected EOF + state = nil; + elseif buffer:length() ~= 0 then -- unexpected EOF error = true; return error_cb("unexpected-eof"); end return; end - if buftable then - t_insert(buf, data); - else - buf = { buf, data }; - buftable = true; - end - buflen = buflen + #data; - if buflen > buflimit then error = true; return error_cb("max-buffer-size-exceeded"); end - while buflen > 0 do + if not buffer:write(data) then error = true; return error_cb("max-buffer-size-exceeded"); end + while buffer:length() > 0 do if state == nil then -- read request - if buftable then buf, buftable = t_concat(buf), false; end - local index = buf:find("\r\n\r\n", nil, true); + local index = buffer:sub(1, headlimit):find("\r\n\r\n", nil, true); if not index then return; end -- not enough data -- FIXME was reason_phrase meant to be passed on somewhere? local method, path, httpversion, status_code, reason_phrase; -- luacheck: ignore reason_phrase local first_line; local headers = {}; - for line in buf:sub(1,index+1):gmatch("([^\r\n]+)\r\n") do -- parse request + for line in buffer:read(index+3):gmatch("([^\r\n]+)\r\n") do -- parse request if first_line then local key, val = line:match("^([^%s:]+): *(.*)$"); if not key then error = true; return error_cb("invalid-header-line"); end -- TODO handle multi-line and invalid headers @@ -101,7 +97,7 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) code = status_code; httpversion = httpversion; headers = headers; - body = have_body and "" or nil; + body = false; -- COMPAT the properties below are deprecated responseversion = httpversion; responseheaders = headers; @@ -126,60 +122,52 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) path = path; httpversion = httpversion; headers = headers; - body = nil; + body = false; + body_sink = nil; }; end - buf = buf:sub(index + 4); - buflen = #buf; + if chunked then + packet.body_buffer = dbuffer.new(buflimit); + end state = true; end if state then -- read body - if client then - if chunked then - if chunk_start and buflen - chunk_start - 2 < chunk_size then - return; - end -- not enough data - if buftable then buf, buftable = t_concat(buf), false; end - if not buf:find("\r\n", nil, true) then - return; - end -- not enough data - if not chunk_size then - chunk_size, chunk_start = buf:match("^(%x+)[^\r\n]*\r\n()"); - chunk_size = chunk_size and tonumber(chunk_size, 16); - if not chunk_size then error = true; return error_cb("invalid-chunk-size"); end - end - if chunk_size == 0 and buf:find("\r\n\r\n", chunk_start-2, true) then - state, chunk_size = nil, nil; - buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped - success_cb(packet); - elseif buflen - chunk_start - 2 >= chunk_size then -- we have a chunk - packet.body = packet.body..buf:sub(chunk_start, chunk_start + (chunk_size-1)); - buf = buf:sub(chunk_start + chunk_size + 2); - buflen = buflen - (chunk_start + chunk_size + 2 - 1); - chunk_size, chunk_start = nil, nil; - else -- Partial chunk remaining - break; + if chunked then + local chunk_header = buffer:sub(1, 512); -- XXX How large do chunk headers grow? + local chunk_size, chunk_start = chunk_header:match("^(%x+)[^\r\n]*\r\n()"); + if not chunk_size then return; end + chunk_size = chunk_size and tonumber(chunk_size, 16); + if not chunk_size then error = true; return error_cb("invalid-chunk-size"); end + if chunk_size == 0 and chunk_header:find("\r\n\r\n", chunk_start-2, true) then + local body_buffer = packet.body_buffer; + if body_buffer then + packet.body_buffer = nil; + body_buffer:collapse(); + packet.body = body_buffer:read_chunk() or ""; end - elseif len and buflen >= len then - if buftable then buf, buftable = t_concat(buf), false; end - if packet.code == 101 then - packet.body, buf, buflen, buftable = buf, {}, 0, true; - else - packet.body, buf = buf:sub(1, len), buf:sub(len + 1); - buflen = #buf; - end - state = nil; success_cb(packet); - else + + buffer:collapse(); + local buf = buffer:read_chunk(); + buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped + buffer:write(buf); + state, chunked = nil, nil; + success_cb(packet); + elseif buffer:length() - chunk_start - 2 >= chunk_size then -- we have a chunk + buffer:discard(chunk_start - 1); -- TODO verify that it's not off-by-one + packet.body_buffer:write(buffer:read(chunk_size)); + buffer:discard(2); -- CRLF + else -- Partial chunk remaining break; end - elseif buflen >= len then - if buftable then buf, buftable = t_concat(buf), false; end - packet.body, buf = buf:sub(1, len), buf:sub(len + 1); - buflen = #buf; + elseif buffer:length() >= len then + assert(not chunked) + packet.body = buffer:read(len) or ""; state = nil; success_cb(packet); else break; end + else + break; end end end; -- cgit v1.2.3 From 91d2ab91086d2aebcbc4d47a5bce05c6cd3abdcb Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 1 Aug 2020 18:41:23 +0200 Subject: net.http.parser: Allow specifying sink for large request bodies This enables uses such as saving uploaded files directly to a file on disk or streaming parsing of payloads. See #726 --- net/http/parser.lua | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) (limited to 'net/http') diff --git a/net/http/parser.lua b/net/http/parser.lua index 84b1e005..b8396518 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -88,8 +88,6 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) if not first_line then error = true; return error_cb("invalid-status-line"); end chunked = have_body and headers["transfer-encoding"] == "chunked"; len = tonumber(headers["content-length"]); -- TODO check for invalid len - if len and len > bodylimit then error = true; return error_cb("content-length-limit-exceeded"); end - -- TODO ask a callback whether to proceed in case of large requests or Expect: 100-continue if client then -- FIXME handle '100 Continue' response (by skipping it) if not have_body then len = 0; end @@ -126,9 +124,17 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) body_sink = nil; }; end - if chunked then + if len and len > bodylimit then + -- Early notification, for redirection + success_cb(packet); + if not packet.body_sink then error = true; return error_cb("content-length-limit-exceeded"); end + end + if chunked and not packet.body_sink then + success_cb(packet); + if not packet.body_sink then packet.body_buffer = dbuffer.new(buflimit); end + end state = true; end if state then -- read body @@ -154,11 +160,23 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) success_cb(packet); elseif buffer:length() - chunk_start - 2 >= chunk_size then -- we have a chunk buffer:discard(chunk_start - 1); -- TODO verify that it's not off-by-one - packet.body_buffer:write(buffer:read(chunk_size)); + (packet.body_sink or packet.body_buffer):write(buffer:read(chunk_size)); buffer:discard(2); -- CRLF else -- Partial chunk remaining break; end + elseif packet.body_sink then + local chunk = buffer:read_chunk(len); + while chunk and len > 0 do + if packet.body_sink:write(chunk) then + len = len - #chunk; + chunk = buffer:read_chunk(len); + else + error = true; + return error_cb("body-sink-write-failure"); + end + end + if len == 0 then state = nil; success_cb(packet); end elseif buffer:length() >= len then assert(not chunked) packet.body = buffer:read(len) or ""; -- cgit v1.2.3 From 9287d929c220bd88fff5458f143f24512800a63e Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Thu, 13 Aug 2020 17:01:05 +0100 Subject: net.http.errors: Add new module for converting net.http errors to util.error objects --- net/http/errors.lua | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 net/http/errors.lua (limited to 'net/http') diff --git a/net/http/errors.lua b/net/http/errors.lua new file mode 100644 index 00000000..ae45d5a8 --- /dev/null +++ b/net/http/errors.lua @@ -0,0 +1,115 @@ +-- This module returns a table that is suitable for use as a util.error registry, +-- and a function to return a util.error object given callback 'code' and 'body' +-- parameters. + +local codes = require "net.http.codes"; +local util_error = require "util.error"; + +local error_templates = { + -- This code is used by us to report a client-side or connection error. + -- Instead of using the code, use the supplied body text to get one of + -- the more detailed errors below. + [0] = { + code = 0, type = "cancel", condition = "internal-server-error"; + text = "Connection or internal error"; + }; + + -- These are net.http built-in errors, they are returned in + -- the body parameter when code == 0 + ["cancelled"] = { + code = 0, type = "cancel", condition = "remote-server-timeout"; + text = "Request cancelled"; + }; + ["connection-closed"] = { + code = 0, type = "wait", condition = "remote-server-timeout"; + text = "Connection closed"; + }; + ["certificate-chain-invalid"] = { + code = 0, type = "cancel", condition = "remote-server-timeout"; + text = "Server certificate not trusted"; + }; + ["certificate-verify-failed"] = { + code = 0, type = "cancel", condition = "remote-server-timeout"; + text = "Server certificate invalid"; + }; + ["connection failed"] = { + code = 0, type = "cancel", condition = "remote-server-not-found"; + text = "Connection failed"; + }; + ["invalid-url"] = { + code = 0, type = "modify", condition = "bad-request"; + text = "Invalid URL"; + }; + + -- This doesn't attempt to map every single HTTP code (not all have sane mappings), + -- but all the common ones should be covered. XEP-0086 was used as reference for + -- most of these. + [400] = { type = "modify", condition = "bad-request" }; + [401] = { type = "auth", condition = "not-authorized" }; + [402] = { type = "auth", condition = "payment-required" }; + [403] = { type = "auth", condition = "forbidden" }; + [404] = { type = "cancel", condition = "item-not-found" }; + [405] = { type = "cancel", condition = "not-allowed" }; + [406] = { type = "modify", condition = "not-acceptable" }; + [407] = { type = "auth", condition = "registration-required" }; + [408] = { type = "wait", condition = "remote-server-timeout" }; + [409] = { type = "cancel", condition = "conflict" }; + [410] = { type = "cancel", condition = "gone" }; + [411] = { type = "modify", condition = "bad-request" }; + [412] = { type = "cancel", condition = "conflict" }; + [413] = { type = "modify", condition = "resource-constraint" }; + [414] = { type = "modify", condition = "resource-constraint" }; + [415] = { type = "cancel", condition = "feature-not-implemented" }; + [416] = { type = "modify", condition = "bad-request" }; + + [422] = { type = "modify", condition = "bad-request" }; + [423] = { type = "wait", condition = "resource-constraint" }; + + [429] = { type = "wait", condition = "resource-constraint" }; + [431] = { type = "modify", condition = "resource-constraint" }; + [451] = { type = "auth", condition = "forbidden" }; + + [500] = { type = "wait", condition = "internal-server-error" }; + [501] = { type = "cancel", condition = "feature-not-implemented" }; + [502] = { type = "wait", condition = "remote-server-timeout" }; + [503] = { type = "cancel", condition = "service-unavailable" }; + [504] = { type = "wait", condition = "remote-server-timeout" }; + [507] = { type = "wait", condition = "resource-constraint" }; + [511] = { type = "auth", condition = "not-authorized" }; +}; + +for k, v in pairs(codes) do + if error_templates[k] then + error_templates[k].code = k; + error_templates[k].text = v; + else + error_templates[k] = { type = "cancel", condition = "undefined-condition", text = v, code = k }; + end +end + +setmetatable(error_templates, { + __index = function(_, k) + if type(k) ~= "number" then + return nil; + end + return { + type = "cancel"; + condition = "undefined-condition"; + text = codes[k] or (k.." Unassigned"); + code = k; + }; + end +}); + +local function new(code, body, context) + if code == 0 then + return util_error.new(body, context, error_templates); + else + return util_error.new(code, context, error_templates); + end +end + +return { + registry = error_templates; + new = new; +}; -- cgit v1.2.3 From 031a8d8e6458cb2c5ba115a75cd4f404326f7acf Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 20 Aug 2020 16:43:27 +0200 Subject: net.http.parser: Fix indentation Probably due to a rebase/merge with a merge tool that ignores whitespace. Happens all the time to me :( --- net/http/parser.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'net/http') diff --git a/net/http/parser.lua b/net/http/parser.lua index b8396518..3470ffb1 100644 --- a/net/http/parser.lua +++ b/net/http/parser.lua @@ -132,8 +132,8 @@ function httpstream.new(success_cb, error_cb, parser_type, options_cb) if chunked and not packet.body_sink then success_cb(packet); if not packet.body_sink then - packet.body_buffer = dbuffer.new(buflimit); - end + packet.body_buffer = dbuffer.new(buflimit); + end end state = true; end -- cgit v1.2.3 From 9229c7a57188168e992145e4f3be6fd381ffe90c Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Mon, 28 Sep 2020 16:21:41 +0100 Subject: net.http.server: Default to HTTP result code 500 when promise is rejected --- net/http/server.lua | 1 + 1 file changed, 1 insertion(+) (limited to 'net/http') diff --git a/net/http/server.lua b/net/http/server.lua index f563d330..0070367a 100644 --- a/net/http/server.lua +++ b/net/http/server.lua @@ -197,6 +197,7 @@ local function handle_result(request, response, result) result:next(function (ret) handle_result(request, response, ret); end, function (err) + response.status_code = 500; handle_result(request, response, err or 500); end); return true; -- cgit v1.2.3